tbb concurrent hash example

IntelPython · Hardcode84 · Feb 21, 2020 · Feb 21, 2020 · Feb 21, 2020 · Feb 28, 2020
commit 1847336bd272121c833ef6e234775a57f73579d9
diff --git a/sdc/_concurrent_hash.cpp b/sdc/_concurrent_hash.cpp
@@ -0,0 +1,137 @@
+//*****************************************************************************
+// Copyright (c) 2020, Intel Corporation All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//    Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+//
+//    Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//*****************************************************************************
+
+#include <Python.h>
+#include <cstdint>
+#include <tbb/concurrent_hash_map.h>
+#include <tbb/concurrent_vector.h>
+
+
+template<typename Key, typename Val>
+using hashmap = tbb::concurrent_hash_map<Key,tbb::concurrent_vector<Val>>;
+
+template<typename Key, typename Val>
+using iter_range = std::pair<typename hashmap<Key, Val>::iterator, typename hashmap<Key, Val>::iterator>;
+
+using int_hashmap = hashmap<int64_t, size_t>;
+using int_hashmap_iters = iter_range<int64_t, size_t>;
+
+extern "C"
+{
+void* create_int_hashmap()
+{
+    return new int_hashmap;
+}
+
+void delete_int_hashmap(void* obj)
+{
+    delete static_cast<int_hashmap*>(obj);
+}
+
+void addelem_int_hashmap(void* obj, int64_t key, size_t val)
+{
+    auto& h = *static_cast<int_hashmap*>(obj);
+    int_hashmap::accessor ac;
+    h.insert(ac, key);
+    auto& vec = ac->second;
+    ac.release();
+    vec.push_back(val);
+    // h[key].push_back(val);
+}
+
+void* createiter_int_hashmap(void* obj)
+{
+    auto& h = *static_cast<int_hashmap*>(obj);
+    return new int_hashmap_iters{h.begin(), h.end()};
+}
+
+int32_t enditer_int_hashmap(void* it)
+{
+    auto& r = *static_cast<int_hashmap_iters*>(it);
+    return static_cast<int32_t>(r.first == r.second);
+}
+
+void nextiter_int_hashmap(void* it)
+{
+    auto& r = *static_cast<int_hashmap_iters*>(it);
+    ++r.first;
+}
+
+int64_t iterkey_int_hashmap(void* it)
+{
+    auto& r = *static_cast<int_hashmap_iters*>(it);
+    return r.first->first;
+}
+
+size_t itersize_int_hashmap(void* it)
+{
+    auto& r = *static_cast<int_hashmap_iters*>(it);
+    return r.first->second.size();
+}
+
+size_t iterelem_int_hashmap(void* it, size_t index)
+{
+    auto& r = *static_cast<int_hashmap_iters*>(it);
+    return r.first->second[index];
+}
+
+void deleteiter_int_hashmap(void* obj)
+{
+    delete static_cast<int_hashmap_iters*>(obj);
+}
+
+
+PyMODINIT_FUNC PyInit_hconcurrent_hash()
+{
+    static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "hconcurrent_hash",
+        "No docs",
+        -1,
+        NULL,
+    };
+    PyObject* m = PyModule_Create(&moduledef);
+    if (m == NULL)
+    {
+        return NULL;
+    }
+
+#define REGISTER(func) PyObject_SetAttrString(m, #func, PyLong_FromVoidPtr((void*)(&func)));
+    REGISTER(create_int_hashmap)
+    REGISTER(delete_int_hashmap)
+    REGISTER(addelem_int_hashmap)
+
+    REGISTER(createiter_int_hashmap)
+    REGISTER(enditer_int_hashmap)
+    REGISTER(nextiter_int_hashmap)
+    REGISTER(iterkey_int_hashmap)
+    REGISTER(itersize_int_hashmap)
+    REGISTER(iterelem_int_hashmap)
+    REGISTER(deleteiter_int_hashmap)
+#undef REGISTER
+    return m;
+}
+}
diff --git a/sdc/concurrent_hash.py b/sdc/concurrent_hash.py
@@ -0,0 +1,116 @@
+import numba
+import sdc
+
+from numba import types, typing, generated_jit
+from numba.extending import models, register_model
+from numba.extending import lower_builtin, overload_method, overload, intrinsic
+
+from llvmlite import ir as lir
+import llvmlite.binding as ll
+from . import hconcurrent_hash
+ll.add_symbol('create_int_hashmap', hconcurrent_hash.create_int_hashmap)
+ll.add_symbol('delete_int_hashmap', hconcurrent_hash.delete_int_hashmap)
+ll.add_symbol('addelem_int_hashmap',hconcurrent_hash.addelem_int_hashmap)
+
+ll.add_symbol('createiter_int_hashmap',hconcurrent_hash.createiter_int_hashmap)
+ll.add_symbol('enditer_int_hashmap',hconcurrent_hash.enditer_int_hashmap)
+ll.add_symbol('nextiter_int_hashmap',hconcurrent_hash.nextiter_int_hashmap)
+ll.add_symbol('iterkey_int_hashmap',hconcurrent_hash.iterkey_int_hashmap)
+ll.add_symbol('itersize_int_hashmap',hconcurrent_hash.itersize_int_hashmap)
+ll.add_symbol('iterelem_int_hashmap',hconcurrent_hash.iterelem_int_hashmap)
+ll.add_symbol('deleteiter_int_hashmap',hconcurrent_hash.deleteiter_int_hashmap)
+
+_create_int_hashmap = types.ExternalFunction("create_int_hashmap",
+                                             types.voidptr())
+_delete_int_hashmap = types.ExternalFunction("delete_int_hashmap",
+                                             types.void(types.voidptr))
+_addelem_int_hashmap = types.ExternalFunction("addelem_int_hashmap",
+                                              types.void(types.voidptr, types.int64, types.intp))
+
+_createiter_int_hashmap = types.ExternalFunction("createiter_int_hashmap",
+                                                 types.voidptr(types.voidptr))
+_enditer_int_hashmap = types.ExternalFunction("enditer_int_hashmap",
+                                              types.int32(types.voidptr))
+_nextiter_int_hashmap = types.ExternalFunction("nextiter_int_hashmap",
+                                               types.void(types.voidptr))
+_iterkey_int_hashmap = types.ExternalFunction("iterkey_int_hashmap",
+                                              types.int64(types.voidptr))
+_itersize_int_hashmap = types.ExternalFunction("itersize_int_hashmap",
+                                               types.intp(types.voidptr))
+_iterelem_int_hashmap = types.ExternalFunction("iterelem_int_hashmap",
+                                               types.intp(types.voidptr, types.intp))
+_deleteiter_int_hashmap = types.ExternalFunction("deleteiter_int_hashmap",
+                                                 types.void(types.voidptr))
+
+
+def create_int_hashmap():
+    pass
+
+def delete_int_hashmap():
+    pass
+
+def addelem_int_hashmap():
+    pass
+
+
+def createiter_int_hashmap():
+    pass
+
+def enditer_int_hashmap():
+    pass
+
+def nextiter_int_hashmap():
+    pass
+
+def iterkey_int_hashmap():
+    pass
+
+def itersize_int_hashmap():
+    pass
+
+def iterelem_int_hashmap():
+    pass
+
+def deleteiter_int_hashmap():
+    pass
+
+@overload(create_int_hashmap)
+def create_int_hashmap_overload():
+    return lambda: _create_int_hashmap()
+
+@overload(delete_int_hashmap)
+def delete_int_hashmap_overload(h):
+    return lambda h: _delete_int_hashmap(h)
+
+@overload(addelem_int_hashmap)
+def addelem_int_hashmap_overload(h, key, val):
+    return lambda h, key, val: _addelem_int_hashmap(h, key, val)
+
+
+@overload(createiter_int_hashmap)
+def createiter_int_hashmap_overload(h):
+    return lambda h: _createiter_int_hashmap(h)
+
+@overload(enditer_int_hashmap)
+def enditer_int_hashmap_overload(h):
+    return lambda h: _enditer_int_hashmap(h)
+
+@overload(nextiter_int_hashmap)
+def nextiter_int_hashmap_overload(h):
+    return lambda h: _nextiter_int_hashmap(h)
+
+@overload(iterkey_int_hashmap)
+def iterkey_int_hashmap_overload(h):
+    return lambda h: _iterkey_int_hashmap(h)
+
+@overload(itersize_int_hashmap)
+def itersize_int_hashmap_overload(h):
+    return lambda h: _itersize_int_hashmap(h)
+
+@overload(iterelem_int_hashmap)
+def iterelem_int_hashmap_overload(h, i):
+    return lambda h, i: _iterelem_int_hashmap(h, i)
+
+@overload(deleteiter_int_hashmap)
+def deleteiter_int_hashmap_overload(h):
+    return lambda h: _deleteiter_int_hashmap(h)
diff --git a/sdc/tests/test_dataframe.py b/sdc/tests/test_dataframe.py
@@ -1791,6 +1791,33 @@ def test_impl():
 
         pd.testing.assert_series_equal(hpat_func(), test_impl())
 
+    def test_tbb(self):
+        import sdc.concurrent_hash
+        def test_impl():
+            h = sdc.concurrent_hash.create_int_hashmap()
+
+            sdc.concurrent_hash.addelem_int_hashmap(h,1,2)
+            sdc.concurrent_hash.addelem_int_hashmap(h,1,3)
+            sdc.concurrent_hash.addelem_int_hashmap(h,1,4)
+            sdc.concurrent_hash.addelem_int_hashmap(h,1,5)
+            sdc.concurrent_hash.addelem_int_hashmap(h,2,6)
+
+            it = sdc.concurrent_hash.createiter_int_hashmap(h)
+            while 0 == sdc.concurrent_hash.enditer_int_hashmap(it):
+                key = sdc.concurrent_hash.iterkey_int_hashmap(it)
+                sz = sdc.concurrent_hash.itersize_int_hashmap(it)
+                for i in range(sz):
+                    val = sdc.concurrent_hash.iterelem_int_hashmap(it, i)
+                    print(key, val)
+
+                sdc.concurrent_hash.nextiter_int_hashmap(it)
+
+            sdc.concurrent_hash.deleteiter_int_hashmap(it)
+            sdc.concurrent_hash.delete_int_hashmap(h)
+
+        hpat_func = self.jit(test_impl)
+        hpat_func()
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/setup.py b/setup.py
@@ -26,6 +26,7 @@
 # *****************************************************************************
 
 from setuptools import setup, Extension, find_packages, Command
+import sys
 import platform
 import os
 from docs.source.buildscripts.sdc_build_doc import SDCBuildDoc
@@ -197,6 +198,59 @@ def readme():
 if _has_opencv:
     _ext_mods.append(ext_cv_wrapper)
 
+# Copypaste from numba
+def check_file_at_path(path2file):
+        """
+        Takes a list as a path, a single glob (*) is permitted as an entry which
+        indicates that expansion at this location is required (i.e. version
+        might not be known).
+        """
+        found = None
+        path2check = [os.path.split(os.path.split(sys.executable)[0])[0]]
+        path2check += [os.getenv(n, '') for n in ['CONDA_PREFIX', 'PREFIX']]
+        if sys.platform.startswith('win'):
+            path2check += [os.path.join(p, 'Library') for p in path2check]
+        for p in path2check:
+            if p:
+                if '*' in path2file:
+                    globloc = path2file.index('*')
+                    searchroot = os.path.join(*path2file[:globloc])
+                    try:
+                        potential_locs = os.listdir(os.path.join(p, searchroot))
+                    except BaseException:
+                        continue
+                    searchfor = path2file[globloc + 1:]
+                    for x in potential_locs:
+                        potpath = os.path.join(p, searchroot, x, *searchfor)
+                        if os.path.isfile(potpath):
+                            found = p  # the latest is used
+                elif os.path.isfile(os.path.join(p, *path2file)):
+                    found = p  # the latest is used
+        return found
+
+# Search for Intel TBB, first check env var TBBROOT then conda locations
+tbb_root = os.getenv('TBBROOT')
+if not tbb_root:
+    tbb_root = check_file_at_path(['include', 'tbb', 'tbb.h'])
+
+print("Using Intel TBB from:", tbb_root)
+ext_hconcurrent_hash = Extension(
+    name="sdc.hconcurrent_hash",
+    sources=["sdc/_concurrent_hash.cpp"],
+    include_dirs=[os.path.join(tbb_root, 'include')],
+    libraries=['tbb'],
+    library_dirs=[
+      # for Linux
+      os.path.join(tbb_root, 'lib', 'intel64', 'gcc4.4'),
+      # for MacOS
+      os.path.join(tbb_root, 'lib'),
+      # for Windows
+      os.path.join(tbb_root, 'lib', 'intel64', 'vc_mt'),
+    ],
+    language="c++",
+    )
+
+_ext_mods.append(ext_hconcurrent_hash)
 
 class style(Command):
     """ Command to check and adjust code style