From 6951079b0bed9082cb6b37c0f5963a3261124b81 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Thu, 30 Jul 2020 16:01:49 +0200 Subject: [PATCH 01/10] added new option: num_multi_probes (number of multi-probes to perform per hash table using multi-probe LSH) --- include/argv_parser.hpp | 3 ++- include/options.hpp | 26 ++++++++++++++++++++++++-- src/main.cpp | 4 ++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/include/argv_parser.hpp b/include/argv_parser.hpp index adc9ad1..b4199e9 100644 --- a/include/argv_parser.hpp +++ b/include/argv_parser.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-28 + * @date 2020-07-30 * * @brief Implements a very simple command line argument parser specifically for this project. */ @@ -50,6 +50,7 @@ class argv_parser { { "num_hash_tables", "number of hash tables to create" }, { "hash_table_size", "size of each hash table (must be a prime)" }, { "num_hash_functions", "number of hash functions per hash table" }, + { "num_multi_probes", "number of multi-probe steps to perform" }, { "w", "constant used in the hash functions" } }; diff --git a/include/options.hpp b/include/options.hpp index 06a3a57..a8ef496 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-30 * * @brief Implements a @ref options class for managing hyperparameters. */ @@ -25,6 +25,7 @@ #include #include +// TODO 2020-07-30 15:48 marcel: maybe change DEBUG assertions to actual exceptions /** * @brief Class containing all hyperparameters to change the behaviour of the algorithm. @@ -92,6 +93,8 @@ struct options : detail::options_base { this->set_hash_table_size(detail::convert_to(value)); } else if (opt == "num_hash_functions") { this->set_num_hash_functions(detail::convert_to(value)); + } else if (opt == "num_multi_probes") { + this->set_num_multi_probes(detail::convert_to(value)); } else if (opt == "w") { this->set_w(detail::convert_to(value)); } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" || opt == "hash_functions_type") { @@ -165,6 +168,18 @@ struct options : detail::options_base { num_hash_functions_ = factory_num_hash_functions; return *this; } + /** + * @brief Set the new number of multi-probes to perform per hash table. + * @param[in] factory_num_multi_probes number of multi-probes + * @return `*this` + * + * @pre @p factory_num_multi_probes **must** be greater than `0`. + */ + factory& set_num_multi_probes(const index_type factory_num_multi_probes) { + DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_multi_probes, "Illegal number of multi-probes!: 0 < {}", factory_num_multi_probes); + num_multi_probes_ = factory_num_multi_probes; + return *this; + } /** * @brief Set the new w value used in the hash value calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. * @param[in] factory_w constant value for the hash value calculation @@ -209,6 +224,7 @@ struct options : detail::options_base { index_type num_hash_tables_ = static_cast(2); hash_value_type hash_table_size_ = static_cast(105613); index_type num_hash_functions_ = static_cast(4); + index_type num_multi_probes_ = static_cast(6); real_type w_ = static_cast(1.0); int comm_rank_; @@ -222,7 +238,10 @@ struct options : detail::options_base { options(options::factory fact) : hash_pool_size(fact.hash_pool_size_), num_cut_off_points(fact.num_cut_off_points_), num_hash_tables(fact.num_hash_tables_), hash_table_size(fact.hash_table_size_), - num_hash_functions(fact.num_hash_functions_), w(fact.w_) { } + num_hash_functions(fact.num_hash_functions_), num_multi_probes(fact.num_multi_probes_), w(fact.w_) + { + DEBUG_ASSERT_MPI(comm_rank_, 0.0 < factory_w, "Illegal 'w' value!: 0.0 < {}", factory_w); + } // ---------------------------------------------------------------------------------------------------------- // // runtime options // @@ -238,6 +257,8 @@ struct options : detail::options_base { const hash_value_type hash_table_size; /// The number of hash functions per hash table. const index_type num_hash_functions; + /// The number of multi-probes to perform per hash table. + const index_type num_multi_probes; /// A constant used in the hash functions: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. const real_type w; @@ -281,6 +302,7 @@ struct options : detail::options_base { out << "num_hash_tables " << opt.num_hash_tables << '\n'; out << "hash_table_size " << opt.hash_table_size << '\n'; out << "num_hash_functions " << opt.num_hash_functions << '\n'; + out << "num_multi_probes " << opt.num_multi_probes << '\n'; out << "w " << opt.w; return out; diff --git a/src/main.cpp b/src/main.cpp index f93327c..3046bf7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -174,6 +174,10 @@ int custom_main(MPI_Comm& communicator, const int argc, char** argv) { options_factory.set_num_hash_functions( parser.argv_as().num_hash_functions)>>("num_hash_functions")); } + if (parser.has_argv("num_multi_probes")) { + options_factory.set_num_multi_probes( + parser.argv_as().num_multi_probes)>>("num_multi_probes")); + } if (parser.has_argv("w")) { options_factory.set_w( parser.argv_as().w)>>("w")); From af0d7ca9b46abedc2c4eb43a14487fe15768812d Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Thu, 30 Jul 2020 16:02:06 +0200 Subject: [PATCH 02/10] added missing option descriptions --- include/argv_parser.hpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/include/argv_parser.hpp b/include/argv_parser.hpp index b4199e9..be3db6b 100644 --- a/include/argv_parser.hpp +++ b/include/argv_parser.hpp @@ -22,18 +22,22 @@ /** * @brief Class to parse command line arguments. * @details The supported command line options are: - * | command line argument | description | - * |:----------------------|:---------------------------------------------------------------------------------------------| - * | help | Prints the help screen. | - * | options | Path to the options file to load. | - * | save_options | Path to the file to save the currently used options to. | - * | data | Path to the data file to load (**required**). | - * | k | The number of nearest-neighbors to search for (**required**). | - * | save_knn | Path to the file to save the found k-nearest-neighbors to. | - * | num_hash_tables | The number of used hash tables. | - * | hash_table_size | The size of each hash table. | - * | num_hash_functions | The number of hash functions to calculate the hash values with. | - * | w | A constant used in the hash functions calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. | + * | command line argument | description | + * |:----------------------|:----------------------------------------------------------------------------------------------------| + * | help | Prints the help screen. | + * | options | Path to the options file to load. | + * | save_options | Path to the file to save the currently used options to. | + * | data | Path to the data file to load (**required**). | + * | k | The number of nearest-neighbors to search for (**required**). | + * | save_knn | Path to the file to save the found k-nearest-neighbors to. | + * | evaluate_knn | Perform evaluation of the calculated k-nearest-neighbors using the given file as reference. | + * | hash_pool_size | The total number of hash tables to generate (currently only used in entropy-based hash functtions). | + * | num_cut_off_points | The number of cut-off points to calculate for the entropy-based hash functions. | + * | num_hash_tables | The number of used hash tables. | + * | hash_table_size | The size of each hash table. | + * | num_hash_functions | The number of hash functions to calculate the hash values with. | + * | num_multi_probes | The number of multi-probes to perform in multi-probe LSH per hash table. | + * | w | A constant used in the hash functions calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. | */ class argv_parser { /// A list of all legal command line options including their description. From 3a618dbf508cafbacfe0ef6599afa8d625f46609 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Thu, 30 Jul 2020 16:13:34 +0200 Subject: [PATCH 03/10] replaced string::append cascade with stringstream --- include/options.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/options.hpp b/include/options.hpp index a8ef496..ee61767 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -100,9 +100,9 @@ struct options : detail::options_base { } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" || opt == "hash_functions_type") { in.ignore(std::numeric_limits::max(), '\n'); } else { - const std::string msg = std::string("Invalid option '").append(opt).append(" ") - .append(value).append("' in file '").append(file).append("'."); - throw std::invalid_argument(msg); + std::stringstream ss; + ss << "Invalid option '" << opt << " " << "' in file '" << file << "'."; + throw std::invalid_argument(ss.str()); } } } From e2477c33f2e2716f8565b9b2e5263c5f7a6c43a9 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Thu, 30 Jul 2020 16:55:35 +0200 Subject: [PATCH 04/10] changed assertions to exceptions (because wrong parameters would crash the program later on and hence the checks should be performed ALWAYS) --- include/options.hpp | 55 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/include/options.hpp b/include/options.hpp index ee61767..9cd9d2c 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -21,11 +21,9 @@ #include #include -#include #include #include -// TODO 2020-07-30 15:48 marcel: maybe change DEBUG assertions to actual exceptions /** * @brief Class containing all hyperparameters to change the behaviour of the algorithm. @@ -113,9 +111,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_hash_pool_size **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_hash_pool_size isn't greater than `0` */ factory& set_hash_pool_size(const index_type factory_hash_pool_size) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_hash_pool_size, "Illegal number of hash functions in hash pool!: 0 < {}", factory_hash_pool_size); + if (factory_hash_pool_size <= 0) { + throw std::invalid_argument("The number of hash functions in the hash pool must be greater than 0!"); + } hash_pool_size_ = factory_hash_pool_size; return *this; } @@ -123,9 +125,15 @@ struct options : detail::options_base { * @brief Set the new number of cut-of points. * @param[in] factory_num_cut_off_points the number of cut-off points used in the entropy-based hash functions * @return `*this` + * + * @pre @p factory_num_cut_off_points **must** be greater than `0` + * + * @throws std::invalid_argument if @p factory_num_cut_off_points isn't greater than `0` */ factory& set_num_cut_off_points(const index_type factory_num_cut_off_points) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_cut_off_points, "Illegal number of cut-off points!: 0 < {}", factory_num_cut_off_points); + if (factory_num_cut_off_points <= 0) { + throw std::invalid_argument("The number of cut-off points used by the entropy-based hash functions must be greater than 0!"); + } num_cut_off_points_ = factory_num_cut_off_points; return *this; } @@ -135,9 +143,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_num_hash_tables **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_hash_tables isn't greater than `0` */ factory& set_num_hash_tables(const index_type factory_num_hash_tables) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_hash_tables, "Illegal number of hash tables!: 0 < {}", factory_num_hash_tables); + if (factory_num_hash_tables <= 0) { + throw std::invalid_argument("The number of hash tables must be greater than 0!"); + } num_hash_tables_ = factory_num_hash_tables; return *this; } @@ -149,10 +161,16 @@ struct options : detail::options_base { * * @pre @p factory_hash_table_size **must** be greater than `0`. * @pre @p factory_hash_table_size **must** be a prime number. + * + * @throws std::invalid_argument if @p factory_hash_table_size isn't greater than `0` or isn't a prime number */ factory& set_hash_table_size(const hash_value_type factory_hash_table_size) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_hash_table_size, "Illegal hash_table_size!: 0 < {}", factory_hash_table_size); - DEBUG_ASSERT_MPI(comm_rank_, this->is_prime(factory_hash_table_size), "{} is not a prime!", factory_hash_table_size); + if (factory_hash_table_size <= 0) { + throw std::invalid_argument("The size of a hash table must be greater than 0!"); + } + if (!this->is_prime(factory_hash_table_size)) { + throw std::invalid_argument("The size of a hash table must be a prime number!"); + } hash_table_size_ = factory_hash_table_size; return *this; } @@ -162,9 +180,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_num_hash_functions **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_hash_functions isn't greater than `0` */ factory& set_num_hash_functions(const index_type factory_num_hash_functions) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_hash_functions, "Illegal number of hash functions!: 0 < {}", factory_num_hash_functions); + if (factory_num_hash_functions <= 0) { + throw std::invalid_argument("The number of hash functions per hash table must be greater than 0!"); + } num_hash_functions_ = factory_num_hash_functions; return *this; } @@ -174,9 +196,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_num_multi_probes **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_multi_probes isn't greater than `0` */ factory& set_num_multi_probes(const index_type factory_num_multi_probes) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_multi_probes, "Illegal number of multi-probes!: 0 < {}", factory_num_multi_probes); + if (factory_num_multi_probes <= 0) { + throw std::invalid_argument("The number of multi-probes per hash table must be greater than 0!"); + } num_multi_probes_ = factory_num_multi_probes; return *this; } @@ -186,9 +212,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_w **must** be greater than `0.0`. + * + * @throws std::invalid_argument if @p factory_w isn't greater than `0.0` */ factory& set_w(const real_type factory_w) { - DEBUG_ASSERT_MPI(comm_rank_, 0.0 < factory_w, "Illegal 'w' value!: 0.0 < {}", factory_w); + if (factory_w <= 0.0) { + throw std::invalid_argument("The value of w must be greater than 0.0!"); + } w_ = factory_w; return *this; } @@ -240,7 +270,10 @@ struct options : detail::options_base { num_hash_tables(fact.num_hash_tables_), hash_table_size(fact.hash_table_size_), num_hash_functions(fact.num_hash_functions_), num_multi_probes(fact.num_multi_probes_), w(fact.w_) { - DEBUG_ASSERT_MPI(comm_rank_, 0.0 < factory_w, "Illegal 'w' value!: 0.0 < {}", factory_w); + // TODO 2020-07-30 16:48 marcel: meaningful restriction? + if (num_multi_probes > num_hash_functions) { + throw std::invalid_argument("The number of multi-probes can't be greater than the number of hash functions!"); + } } // ---------------------------------------------------------------------------------------------------------- // From 3ee0fd1314773708b2632a53da63c1f6bc7cfb70 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Fri, 31 Jul 2020 11:54:17 +0200 Subject: [PATCH 05/10] replaced unnecessary sort with inplace_merge --- include/entropy_based_hash_function.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/entropy_based_hash_function.hpp b/include/entropy_based_hash_function.hpp index 75dbf37..2562739 100644 --- a/include/entropy_based_hash_function.hpp +++ b/include/entropy_based_hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-30 + * @date 2020-07-31 * * @brief Implements the @ref entropy_based_hash_functions class representing the used entropy-based LSH hash functions */ @@ -41,7 +41,7 @@ void pairwise_exchange(std::vector& a, const int sendrank, const int std::copy(a.begin(), a.end(), all.begin()); std::copy(remote.begin(), remote.end(), all.begin() + a.size()); - std::sort(all.begin(), all.end()); + std::inplace_merge(all.begin(), all.begin() + a.size(), all.end()); std::size_t theirstart = sendrank > comm_rank ? a.size() : 0; std::size_t mystart = sendrank > comm_rank ? 0 : a.size(); From 1a35fbb3c75447fef7b9727deb7082a7a00dc257 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Fri, 31 Jul 2020 12:22:37 +0200 Subject: [PATCH 06/10] added new compile time option: probing_type tag --- include/hash_function.hpp | 44 ++++++++++++++++++++++++++++++++------- include/options.hpp | 15 +++++++++---- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/include/hash_function.hpp b/include/hash_function.hpp index b0d6c01..bff0bca 100644 --- a/include/hash_function.hpp +++ b/include/hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-31 * * @brief Implements the factory functions for the hash functions classes. */ @@ -31,24 +31,52 @@ struct hash_functions { */ static struct RandomProjection{} random_projection; }; - /** * @brief Stream insertion operator overload for the @ref hash_functions::EntropyBased tag. * @param[in] out the output stream - * @return `out` + * @return the output stream */ std::ostream& operator<<(std::ostream& out, hash_functions::EntropyBased) { - out << "entropy_based"; - return out; + return out << "entropy_based"; } /** * @brief Stream insertion operator overload for the @ref hash_functions::RandomProjection tag. * @param[in] out the output stream - * @return `out` + * @return the output stream */ std::ostream& operator<<(std::ostream& out, hash_functions::RandomProjection) { - out << "random_projections"; - return out; + return out << "random_projections"; +} + + +/** + * @brief Struct to encapsulate all possible hash functions probing tags. + */ +struct probing { + /** + * @brief Tag for the LSH using single (normal) probing. + */ + static struct Single{} single; + /** + * @brief Tag for the LSH using multi-probing. + */ + static struct Multiple{} multiple; +}; +/** + * @brief Stream insertion operator overload for the @ref probing::Single tag. + * @param[in] out the output stream + * @return the output stream + */ +std::ostream& operator<<(std::ostream& out, probing::Single) { + return out << "single"; +} +/** + * @brief Stream insertion operator overload for the @ref probing::Multiple tag. + * @param[in] out the output stream + * @return the output stream + */ +std::ostream& operator<<(std::ostream& out, probing::Multiple) { + return out << "multiple"; } diff --git a/include/options.hpp b/include/options.hpp index 9cd9d2c..a485c61 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-30 + * @date 2020-07-31 * * @brief Implements a @ref options class for managing hyperparameters. */ @@ -95,7 +95,9 @@ struct options : detail::options_base { this->set_num_multi_probes(detail::convert_to(value)); } else if (opt == "w") { this->set_w(detail::convert_to(value)); - } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" || opt == "hash_functions_type") { + } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" + || opt == "hash_functions_type" || opt == "probing_type") + { in.ignore(std::numeric_limits::max(), '\n'); } else { std::stringstream ss; @@ -271,8 +273,10 @@ struct options : detail::options_base { num_hash_functions(fact.num_hash_functions_), num_multi_probes(fact.num_multi_probes_), w(fact.w_) { // TODO 2020-07-30 16:48 marcel: meaningful restriction? - if (num_multi_probes > num_hash_functions) { - throw std::invalid_argument("The number of multi-probes can't be greater than the number of hash functions!"); + if constexpr (std::is_same_v) { + if (num_multi_probes > num_hash_functions) { + throw std::invalid_argument("The number of multi-probes can't be greater than the number of hash functions!"); + } } } @@ -301,6 +305,8 @@ struct options : detail::options_base { /// The type of the hash functions. static constexpr auto hash_functions_type = hash_functions::random_projection; + /// Specifies whether multi-probe LSH should be used. + static constexpr auto probing_type = probing::single; /** @@ -330,6 +336,7 @@ struct options : detail::options_base { out << "index_type '" << boost::typeindex::type_id().pretty_name() << "'\n"; out << "hash_value_type '" << boost::typeindex::type_id().pretty_name() << "'\n"; out << "hash_functions_type '" << options::hash_functions_type << "'\n"; + out << "probing_type '" << options::probing_type << "'\n"; out << "hash_pool_size " << opt.hash_pool_size << '\n'; out << "num_cut_off_points " << opt.num_cut_off_points << '\n'; out << "num_hash_tables " << opt.num_hash_tables << '\n'; From 3c2cf9246585cd6b579a29fe4b1e581758c21e76 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Fri, 31 Jul 2020 13:55:06 +0200 Subject: [PATCH 07/10] first (really dump) multi-probe implementation --- include/hash_table.hpp | 75 +++++++++++---------- include/random_projection_hash_function.hpp | 12 +++- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/include/hash_table.hpp b/include/hash_table.hpp index 572cf91..50513da 100644 --- a/include/hash_table.hpp +++ b/include/hash_table.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-31 * * @brief Implements the @ref hash_tables class representing the used LSH hash tables. */ @@ -98,6 +98,7 @@ class hash_tables : detail::hash_tables_base { index_type* nearest_neighbors = new index_type[k]; real_type* distances = new real_type[k]; + // initialize arrays for (index_type i = 0; i < k; ++i) { @@ -114,40 +115,46 @@ class hash_tables : detail::hash_tables_base { } for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - const hash_value_type hash_bucket = hash_function.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data); - - for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; - bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; - ++bucket_element) - { - const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; - const index_type point_idx = point % data.rank_size; - real_type dist = 0.0; - for (index_type dim = 0; dim < data.dims; ++dim) { - const index_type x_idx = data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims); - const real_type x = acc_data_received[x_idx]; - const index_type y_idx = data.get_linear_id(comm_rank_, point_idx, data.rank_size, dim, data.dims); - const real_type y = acc_data_owned[y_idx]; - - dist += (x - y) * (x - y); - } - - // updated nearest-neighbors - const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { - if (first_round && point_idx == idx) return false; - for (index_type i = 0; i < k; ++i) { - if (neighbors[i] == point) return false; + int probes = 0; + if constexpr (std::is_same_v, probing::Multiple>) { + probes = static_cast(opt.num_hash_functions); + } + for (int i = -1; i < probes; ++i) { + const hash_value_type hash_bucket = hash_function.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data, i); + + for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; + bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; + ++bucket_element) + { + const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; + const index_type point_idx = point % data.rank_size; + real_type dist = 0.0; + for (index_type dim = 0; dim < data.dims; ++dim) { + const index_type x_idx = data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims); + const real_type x = acc_data_received[x_idx]; + const index_type y_idx = data.get_linear_id(comm_rank_, point_idx, data.rank_size, dim, data.dims); + const real_type y = acc_data_owned[y_idx]; + + dist += (x - y) * (x - y); } - return true; - }; - if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { - nearest_neighbors[argmax] = point; - distances[argmax] = dist; - max_distance = dist; - for (index_type i = 0; i < k; ++i) { - if (distances[i] > max_distance) { - max_distance = distances[i]; - argmax = i; + + // updated nearest-neighbors + const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { + if (first_round && point_idx == idx) return false; + for (index_type i = 0; i < k; ++i) { + if (neighbors[i] == point) return false; + } + return true; + }; + if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { + nearest_neighbors[argmax] = point; + distances[argmax] = dist; + max_distance = dist; + for (index_type i = 0; i < k; ++i) { + if (distances[i] > max_distance) { + max_distance = distances[i]; + argmax = i; + } } } } diff --git a/include/random_projection_hash_function.hpp b/include/random_projection_hash_function.hpp index a01b5ef..3517979 100644 --- a/include/random_projection_hash_function.hpp +++ b/include/random_projection_hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-31 * * @brief Implements the @ref random_projection_hash_functions class representing the used LSH hash functions. */ @@ -68,7 +68,7 @@ class random_projection_hash_functions : detail::hash_functions_base { [[nodiscard]] static constexpr hash_value_type hash([[maybe_unused]] const int comm_rank, const index_type hash_table, const index_type point, AccData& acc_data, AccHashFunctions& acc_hash_functions, - const Options& opt, const Data& data) + const Options& opt, const Data& data, const index_type pert = -1) { DEBUG_ASSERT_MPI(comm_rank, 0 <= hash_table && hash_table < opt.num_hash_tables, "Out-of-bounce access!: 0 <= {} < {}", hash_table, opt.num_hash_tables); @@ -82,7 +82,13 @@ class random_projection_hash_functions : detail::hash_functions_base { hash += acc_data[data_type::get_linear_id(comm_rank, point, data.rank_size, dim, data.dims)] * acc_hash_functions[get_linear_id(comm_rank, hash_table, hash_function, dim, opt, data)]; } - combined_hash ^= static_cast(hash / opt.w) + hash_value_type bucket = static_cast(hash / opt.w); + if (hash_function == pert) { + hash_value_type next_bucket = bucket + 1; + real_type middle = (next_bucket - bucket) / 2.0 * opt.w; + bucket = hash < middle ? bucket - 1 : bucket + 1; + } + combined_hash ^= bucket + static_cast(0x9e3779b9) + (combined_hash << static_cast(6)) + (combined_hash >> static_cast(2)); From 938fe90f7e33fd65fe474ac0b40e33c07f767ce1 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Tue, 4 Aug 2020 15:31:12 +0200 Subject: [PATCH 08/10] tried "better" multi-probe implementation --- include/hash_table.hpp | 84 +++++++++++++++++---- include/random_projection_hash_function.hpp | 81 +++++++++++++++----- 2 files changed, 131 insertions(+), 34 deletions(-) diff --git a/include/hash_table.hpp b/include/hash_table.hpp index 50513da..bf6c8b8 100644 --- a/include/hash_table.hpp +++ b/include/hash_table.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-31 + * @date 2020-08-04 * * @brief Implements the @ref hash_tables class representing the used LSH hash tables. */ @@ -20,6 +20,14 @@ #include +struct pair_type { + pair_type() = default; + pair_type(const std::uint32_t idx, const std::int32_t delta) : idx(idx), delta(delta) { } + + std::uint32_t idx = 0; + std::int32_t delta = 0.0; +}; + /** * @brief Class representing the hash tables used in the LSH algorithm. * @tparam layout determines whether the hash functions are saved as *Array of Structs* or *Struct of Arrays* @@ -44,7 +52,13 @@ class hash_tables : detail::hash_tables_base { /// The SYCL buffer holding the hash bucket offsets: `offsets.get_count() == options::num_hash_tables * (options::hash_table_size + 1)`. sycl::buffer offsets; /// Hash functions used by this hash tables. - HashFunctions hash_function; + HashFunctions hash_functions; + + + void execute() { + + } + template void calculate_knn(const index_type k, Knns& knns) { @@ -83,7 +97,7 @@ class hash_tables : detail::hash_tables_base { auto acc_data_owned = data_.buffer.template get_access(cgh); auto acc_data_received = data_buffer.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_offsets = offsets.template get_access(cgh); auto acc_hash_tables = buffer.template get_access(cgh); auto acc_knns = knn_buffers.template get_access(cgh); @@ -96,8 +110,11 @@ class hash_tables : detail::hash_tables_base { if (idx >= data.rank_size) return; - index_type* nearest_neighbors = new index_type[k]; - real_type* distances = new real_type[k]; + auto* nearest_neighbors = new index_type[k]; + auto* distances = new real_type[k]; + + auto* probes = new pair_type[opt.num_multi_probes]; + auto* probe_distances = new real_type[opt.num_multi_probes]; // initialize arrays @@ -115,12 +132,47 @@ class hash_tables : detail::hash_tables_base { } for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - int probes = 0; - if constexpr (std::is_same_v, probing::Multiple>) { - probes = static_cast(opt.num_hash_functions); + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + probe_distances[i] = std::numeric_limits::max(); } - for (int i = -1; i < probes; ++i) { - const hash_value_type hash_bucket = hash_function.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data, i); + + index_type probe_argmax = 0; + real_type probe_max = probe_distances[probe_argmax]; + + const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hf, const std::uint32_t delta) { + if (probe_max > slot_dist) { + probes[probe_argmax] = pair_type(hf, delta); + probe_distances[probe_argmax] = slot_dist; + probe_max = slot_dist; + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + if (probe_distances[i] > probe_max) { + probe_max = probe_distances[i]; + probe_argmax = i; + } + } + } + }; + + // calculate distances + for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { + real_type hash = acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, data.dims, opt, data)]; + for (index_type dim = 0; dim < data.dims; ++dim) { + hash += acc_data_received[data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims)] * + acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, dim, opt, data)]; + } + + const auto slot = static_cast(hash / opt.w); + update_probing_sequence(hash - slot * opt.w, hash_function, -1); + update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); + } + +// if constexpr (std::is_same_v, probing::Multiple>) { +// probes = static_cast(opt.num_hash_functions); +// } + for (int probe = -1; probe < static_cast(opt.num_multi_probes); ++probe) { + const auto probe_value = probe == -1 ? pair_type{0, 0} : probes[probe]; + const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, + acc_hash_functions, opt, data, probe_value.idx, probe_value.delta); for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; @@ -169,6 +221,8 @@ class hash_tables : detail::hash_tables_base { delete[] nearest_neighbors; delete[] distances; + delete[] probes; + delete[] probe_distances; }); }); END_TIMING_MPI_AND_BARRIER(calculate_nearest_neighbors, comm_rank_, queue_); @@ -214,7 +268,7 @@ class hash_tables : detail::hash_tables_base { */ hash_tables(sycl::queue& queue, const Options& opt, Data& data, HashFunctions hash_functions, const int comm_rank) : buffer(opt.num_hash_tables * data.rank_size), offsets(opt.num_hash_tables * (opt.hash_table_size + 1)), - hash_function(hash_functions), queue_(queue), comm_rank_(comm_rank), opt_(opt), data_(data) + hash_functions(hash_functions), queue_(queue), comm_rank_(comm_rank), opt_(opt), data_(data) { { // create temporary buffer to count the occurrence of each hash value @@ -250,7 +304,7 @@ class hash_tables : detail::hash_tables_base { START_TIMING(count_hash_values); queue_.submit([&](sycl::handler& cgh) { auto acc_hash_value_count = hash_value_count.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_data = data_.buffer.template get_access(cgh); auto opt = opt_; auto data = data_; @@ -263,7 +317,7 @@ class hash_tables : detail::hash_tables_base { for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { const hash_value_type hash_value = - hash_function.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); + hash_functions.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); acc_hash_value_count[hash_table * opt.hash_table_size + hash_value].fetch_add(1); } }); @@ -307,7 +361,7 @@ class hash_tables : detail::hash_tables_base { START_TIMING(fill_hash_tables); queue_.submit([&](sycl::handler& cgh) { auto acc_data = data_.buffer.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_offsets = offsets.template get_access(cgh); auto acc_hash_tables = buffer.template get_access(cgh); auto opt = opt_; @@ -318,7 +372,7 @@ class hash_tables : detail::hash_tables_base { const index_type idx = item.get_linear_id(); for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - const hash_value_type hash_value = hash_function.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); + const hash_value_type hash_value = hash_functions.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); acc_hash_tables[hash_table * data.rank_size + acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_value + 1].fetch_add(1)] = idx + comm_rank * data.rank_size; } diff --git a/include/random_projection_hash_function.hpp b/include/random_projection_hash_function.hpp index 3517979..2704457 100644 --- a/include/random_projection_hash_function.hpp +++ b/include/random_projection_hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-31 + * @date 2020-08-04 * * @brief Implements the @ref random_projection_hash_functions class representing the used LSH hash functions. */ @@ -68,7 +68,7 @@ class random_projection_hash_functions : detail::hash_functions_base { [[nodiscard]] static constexpr hash_value_type hash([[maybe_unused]] const int comm_rank, const index_type hash_table, const index_type point, AccData& acc_data, AccHashFunctions& acc_hash_functions, - const Options& opt, const Data& data, const index_type pert = -1) + const Options& opt, const Data& data, const index_type idx = 0, const int delta = 0) { DEBUG_ASSERT_MPI(comm_rank, 0 <= hash_table && hash_table < opt.num_hash_tables, "Out-of-bounce access!: 0 <= {} < {}", hash_table, opt.num_hash_tables); @@ -82,13 +82,8 @@ class random_projection_hash_functions : detail::hash_functions_base { hash += acc_data[data_type::get_linear_id(comm_rank, point, data.rank_size, dim, data.dims)] * acc_hash_functions[get_linear_id(comm_rank, hash_table, hash_function, dim, opt, data)]; } - hash_value_type bucket = static_cast(hash / opt.w); - if (hash_function == pert) { - hash_value_type next_bucket = bucket + 1; - real_type middle = (next_bucket - bucket) / 2.0 * opt.w; - bucket = hash < middle ? bucket - 1 : bucket + 1; - } - combined_hash ^= bucket + auto slot = static_cast(hash / opt.w); + combined_hash ^= (hash_function == idx ? slot + delta : slot) + static_cast(0x9e3779b9) + (combined_hash << static_cast(6)) + (combined_hash >> static_cast(2)); @@ -183,7 +178,6 @@ class random_projection_hash_functions : detail::hash_functions_base { template friend class random_projection_hash_functions; - /** * @brief Construct new hash functions. * @param[in] opt the @ref options object representing the currently set options @@ -198,17 +192,66 @@ class random_projection_hash_functions : detail::hash_functions_base { for (std::size_t i = 0; i < tmp_buffer.size(); ++i) { acc[i] = tmp_buffer[i]; } +// +// sycl::queue queue(sycl::default_selector{}); +// queue.submit([&](sycl::handler& cgh) { +// auto acc_probing_sequence = probing_sequence.template get_access(cgh); +// auto acc_data = data.buffer.template get_access(cgh); +// auto acc_hash_functions = data.buffer.template get_access(cgh); +// auto data = data_; +// auto opt = opt_; +// auto comm_rank = comm_rank_; +// +// cgh.parallel_for(sycl::range<>(data.rank_size), [=](sycl::item<> item) { +// const index_type idx = item.get_linear_id(); +// +// auto* probes = new pair_type[opt.num_multi_probes]; +// auto* dists = new real_type[opt.num_multi_probes]; +// +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// dists[i] = std::numeric_limits::max(); +// } +// +// index_type argmax = 0; +// real_type max = dists[argmax]; +// +// const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hash_function, const signed_type delta) { +// if (max > slot_dist) { +// probes[argmax] = pair_type(hash_function, delta); +// dists[argmax] = slot_dist; +// max = slot_dist; +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// if (dists[i] > max) { +// max = dists[i]; +// argmax = i; +// } +// } +// } +// }; +// +// // calculate distances +// for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { +// real_type hash = acc_hash_functions[get_linear_id(comm_rank, 0, hash_function, data.dims, opt, data)]; +// for (index_type dim = 0; dim < data.dims; ++dim) { +// hash += acc_data[data_type::get_linear_id(comm_rank, idx, data.rank_size, dim, data.dims)] * +// acc_hash_functions[get_linear_id(comm_rank, 0, hash_function, dim, opt, data)]; +// } +// +// const auto slot = static_cast(hash / opt.w); +// update_probing_sequence(hash - slot * opt.w, hash_function, -1); +// update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); +// } +// +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// acc_probing_sequence[idx * opt.num_multi_probes + i] = probes[i]; +// } +// +// delete[] probes; +// delete[] dists; +// }); +// }); } - /** - * @brief Construct an empty hash functions buffer. - * @param[in] opt the @ref options object representing the currently set options - * @param[in] data the @ref data object representing the used data set - * @param[in] size the size of the empty buffer - * @param[in] comm_rank the current MPI rank - */ - random_projection_hash_functions(const Options& opt, Data& data, const index_type size, const int comm_rank) - : buffer(size), comm_rank_(comm_rank), opt_(opt), data_(data) { } /// The current MPI rank. const int comm_rank_; From 22cf8ca3c07dc09cf0e103f0a3c869925de77292 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Tue, 4 Aug 2020 15:50:02 +0200 Subject: [PATCH 09/10] refactored multi probing --- include/hash_table.hpp | 159 ++++++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/include/hash_table.hpp b/include/hash_table.hpp index bf6c8b8..9b8e70b 100644 --- a/include/hash_table.hpp +++ b/include/hash_table.hpp @@ -55,8 +55,47 @@ class hash_tables : detail::hash_tables_base { HashFunctions hash_functions; - void execute() { - + template + void execute(const bool first_round, real_type& max_distance, index_type* nearest_neighbors, const index_type k, index_type& argmax, real_type* distances, + const index_type hash_table, const hash_value_type hash_bucket, const index_type idx, const Data& data, const Options& opt, const int comm_rank, + AccOffsets& acc_offsets, AccHashTables& acc_hash_tables, AccDataReceived& acc_data_received, AccDataOwned& acc_data_owned) + { + for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; + bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; + ++bucket_element) + { + const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; + const index_type point_idx = point % data.rank_size; + real_type dist = 0.0; + for (index_type dim = 0; dim < data.dims; ++dim) { + const index_type x_idx = data.get_linear_id(comm_rank, idx, data.rank_size, dim, data.dims); + const real_type x = acc_data_received[x_idx]; + const index_type y_idx = data.get_linear_id(comm_rank, point_idx, data.rank_size, dim, data.dims); + const real_type y = acc_data_owned[y_idx]; + + dist += (x - y) * (x - y); + } + + // updated nearest-neighbors + const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { + if (first_round && point_idx == idx) return false; + for (index_type i = 0; i < k; ++i) { + if (neighbors[i] == point) return false; + } + return true; + }; + if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { + nearest_neighbors[argmax] = point; + distances[argmax] = dist; + max_distance = dist; + for (index_type i = 0; i < k; ++i) { + if (distances[i] > max_distance) { + max_distance = distances[i]; + argmax = i; + } + } + } + } } @@ -113,9 +152,12 @@ class hash_tables : detail::hash_tables_base { auto* nearest_neighbors = new index_type[k]; auto* distances = new real_type[k]; - auto* probes = new pair_type[opt.num_multi_probes]; - auto* probe_distances = new real_type[opt.num_multi_probes]; - + pair_type* probes = nullptr; + real_type* probe_distances = nullptr; + if constexpr (std::is_same_v, probing::Multiple>) { + probes = new pair_type[opt.num_multi_probes]; + probe_distances = new real_type[opt.num_multi_probes]; + } // initialize arrays for (index_type i = 0; i < k; ++i) { @@ -132,83 +174,50 @@ class hash_tables : detail::hash_tables_base { } for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - for (index_type i = 0; i < opt.num_multi_probes; ++i) { - probe_distances[i] = std::numeric_limits::max(); - } + const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data); + execute(first_round, max_distance, nearest_neighbors, k, argmax, distances, hash_table, hash_bucket, idx, data, opt, + comm_rank_, acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned); - index_type probe_argmax = 0; - real_type probe_max = probe_distances[probe_argmax]; - - const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hf, const std::uint32_t delta) { - if (probe_max > slot_dist) { - probes[probe_argmax] = pair_type(hf, delta); - probe_distances[probe_argmax] = slot_dist; - probe_max = slot_dist; - for (index_type i = 0; i < opt.num_multi_probes; ++i) { - if (probe_distances[i] > probe_max) { - probe_max = probe_distances[i]; - probe_argmax = i; - } - } - } - }; - - // calculate distances - for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { - real_type hash = acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, data.dims, opt, data)]; - for (index_type dim = 0; dim < data.dims; ++dim) { - hash += acc_data_received[data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims)] * - acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, dim, opt, data)]; + if constexpr (std::is_same_v, probing::Multiple>) { + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + probe_distances[i] = std::numeric_limits::max(); } - const auto slot = static_cast(hash / opt.w); - update_probing_sequence(hash - slot * opt.w, hash_function, -1); - update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); - } + index_type probe_argmax = 0; + real_type probe_max = probe_distances[probe_argmax]; + + const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hf, const std::uint32_t delta) { + if (probe_max > slot_dist) { + probes[probe_argmax] = pair_type(hf, delta); + probe_distances[probe_argmax] = slot_dist; + probe_max = slot_dist; + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + if (probe_distances[i] > probe_max) { + probe_max = probe_distances[i]; + probe_argmax = i; + } + } + } + }; -// if constexpr (std::is_same_v, probing::Multiple>) { -// probes = static_cast(opt.num_hash_functions); -// } - for (int probe = -1; probe < static_cast(opt.num_multi_probes); ++probe) { - const auto probe_value = probe == -1 ? pair_type{0, 0} : probes[probe]; - const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, - acc_hash_functions, opt, data, probe_value.idx, probe_value.delta); - - for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; - bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; - ++bucket_element) - { - const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; - const index_type point_idx = point % data.rank_size; - real_type dist = 0.0; + // calculate distances + for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { + real_type hash = acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, data.dims, opt, data)]; for (index_type dim = 0; dim < data.dims; ++dim) { - const index_type x_idx = data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims); - const real_type x = acc_data_received[x_idx]; - const index_type y_idx = data.get_linear_id(comm_rank_, point_idx, data.rank_size, dim, data.dims); - const real_type y = acc_data_owned[y_idx]; - - dist += (x - y) * (x - y); + hash += acc_data_received[data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims)] * + acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, dim, opt, data)]; } - // updated nearest-neighbors - const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { - if (first_round && point_idx == idx) return false; - for (index_type i = 0; i < k; ++i) { - if (neighbors[i] == point) return false; - } - return true; - }; - if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { - nearest_neighbors[argmax] = point; - distances[argmax] = dist; - max_distance = dist; - for (index_type i = 0; i < k; ++i) { - if (distances[i] > max_distance) { - max_distance = distances[i]; - argmax = i; - } - } - } + const auto slot = static_cast(hash / opt.w); + update_probing_sequence(hash - slot * opt.w, hash_function, -1); + update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); + } + + for (index_type probe = 0; probe < opt.num_multi_probes; ++probe) { + const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, + acc_hash_functions, opt, data, probes[probe].idx, probes[probe].delta); + execute(first_round, max_distance, nearest_neighbors, k, argmax, distances, hash_table, hash_bucket, idx, data, opt, + comm_rank_, acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned); } } } From 06b4f37dc9d05f63edbd90c768a4244e8be834b1 Mon Sep 17 00:00:00 2001 From: Marcel Breyer Date: Tue, 4 Aug 2020 16:11:18 +0200 Subject: [PATCH 10/10] reordered parameters --- include/hash_table.hpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/hash_table.hpp b/include/hash_table.hpp index 9b8e70b..a91aa06 100644 --- a/include/hash_table.hpp +++ b/include/hash_table.hpp @@ -54,11 +54,12 @@ class hash_tables : detail::hash_tables_base { /// Hash functions used by this hash tables. HashFunctions hash_functions; - + template - void execute(const bool first_round, real_type& max_distance, index_type* nearest_neighbors, const index_type k, index_type& argmax, real_type* distances, - const index_type hash_table, const hash_value_type hash_bucket, const index_type idx, const Data& data, const Options& opt, const int comm_rank, - AccOffsets& acc_offsets, AccHashTables& acc_hash_tables, AccDataReceived& acc_data_received, AccDataOwned& acc_data_owned) + void execute(const int comm_rank, const bool first_round, index_type* nearest_neighbors, real_type* distances, real_type& max_distance, index_type& argmax, + const index_type k, const index_type hash_table, const hash_value_type hash_bucket, const index_type idx, + AccOffsets& acc_offsets, AccHashTables& acc_hash_tables, AccDataReceived& acc_data_received, AccDataOwned& acc_data_owned, + const Data& data, const Options& opt) { for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; @@ -175,8 +176,8 @@ class hash_tables : detail::hash_tables_base { for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data); - execute(first_round, max_distance, nearest_neighbors, k, argmax, distances, hash_table, hash_bucket, idx, data, opt, - comm_rank_, acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned); + execute(comm_rank_, first_round, nearest_neighbors, distances, max_distance, argmax, k, hash_table, hash_bucket, idx, + acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned, data, opt); if constexpr (std::is_same_v, probing::Multiple>) { for (index_type i = 0; i < opt.num_multi_probes; ++i) { @@ -216,8 +217,8 @@ class hash_tables : detail::hash_tables_base { for (index_type probe = 0; probe < opt.num_multi_probes; ++probe) { const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data, probes[probe].idx, probes[probe].delta); - execute(first_round, max_distance, nearest_neighbors, k, argmax, distances, hash_table, hash_bucket, idx, data, opt, - comm_rank_, acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned); + execute(comm_rank_, first_round, nearest_neighbors, distances, max_distance, argmax, k, hash_table, hash_bucket, idx, + acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned, data, opt); } } }