diff --git a/include/argv_parser.hpp b/include/argv_parser.hpp index adc9ad1..be3db6b 100644 --- a/include/argv_parser.hpp +++ b/include/argv_parser.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-28 + * @date 2020-07-30 * * @brief Implements a very simple command line argument parser specifically for this project. */ @@ -22,18 +22,22 @@ /** * @brief Class to parse command line arguments. * @details The supported command line options are: - * | command line argument | description | - * |:----------------------|:---------------------------------------------------------------------------------------------| - * | help | Prints the help screen. | - * | options | Path to the options file to load. | - * | save_options | Path to the file to save the currently used options to. | - * | data | Path to the data file to load (**required**). | - * | k | The number of nearest-neighbors to search for (**required**). | - * | save_knn | Path to the file to save the found k-nearest-neighbors to. | - * | num_hash_tables | The number of used hash tables. | - * | hash_table_size | The size of each hash table. | - * | num_hash_functions | The number of hash functions to calculate the hash values with. | - * | w | A constant used in the hash functions calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. | + * | command line argument | description | + * |:----------------------|:----------------------------------------------------------------------------------------------------| + * | help | Prints the help screen. | + * | options | Path to the options file to load. | + * | save_options | Path to the file to save the currently used options to. | + * | data | Path to the data file to load (**required**). | + * | k | The number of nearest-neighbors to search for (**required**). | + * | save_knn | Path to the file to save the found k-nearest-neighbors to. | + * | evaluate_knn | Perform evaluation of the calculated k-nearest-neighbors using the given file as reference. | + * | hash_pool_size | The total number of hash tables to generate (currently only used in entropy-based hash functtions). | + * | num_cut_off_points | The number of cut-off points to calculate for the entropy-based hash functions. | + * | num_hash_tables | The number of used hash tables. | + * | hash_table_size | The size of each hash table. | + * | num_hash_functions | The number of hash functions to calculate the hash values with. | + * | num_multi_probes | The number of multi-probes to perform in multi-probe LSH per hash table. | + * | w | A constant used in the hash functions calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. | */ class argv_parser { /// A list of all legal command line options including their description. @@ -50,6 +54,7 @@ class argv_parser { { "num_hash_tables", "number of hash tables to create" }, { "hash_table_size", "size of each hash table (must be a prime)" }, { "num_hash_functions", "number of hash functions per hash table" }, + { "num_multi_probes", "number of multi-probe steps to perform" }, { "w", "constant used in the hash functions" } }; diff --git a/include/entropy_based_hash_function.hpp b/include/entropy_based_hash_function.hpp index 75dbf37..2562739 100644 --- a/include/entropy_based_hash_function.hpp +++ b/include/entropy_based_hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-30 + * @date 2020-07-31 * * @brief Implements the @ref entropy_based_hash_functions class representing the used entropy-based LSH hash functions */ @@ -41,7 +41,7 @@ void pairwise_exchange(std::vector& a, const int sendrank, const int std::copy(a.begin(), a.end(), all.begin()); std::copy(remote.begin(), remote.end(), all.begin() + a.size()); - std::sort(all.begin(), all.end()); + std::inplace_merge(all.begin(), all.begin() + a.size(), all.end()); std::size_t theirstart = sendrank > comm_rank ? a.size() : 0; std::size_t mystart = sendrank > comm_rank ? 0 : a.size(); diff --git a/include/hash_function.hpp b/include/hash_function.hpp index b0d6c01..bff0bca 100644 --- a/include/hash_function.hpp +++ b/include/hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-31 * * @brief Implements the factory functions for the hash functions classes. */ @@ -31,24 +31,52 @@ struct hash_functions { */ static struct RandomProjection{} random_projection; }; - /** * @brief Stream insertion operator overload for the @ref hash_functions::EntropyBased tag. * @param[in] out the output stream - * @return `out` + * @return the output stream */ std::ostream& operator<<(std::ostream& out, hash_functions::EntropyBased) { - out << "entropy_based"; - return out; + return out << "entropy_based"; } /** * @brief Stream insertion operator overload for the @ref hash_functions::RandomProjection tag. * @param[in] out the output stream - * @return `out` + * @return the output stream */ std::ostream& operator<<(std::ostream& out, hash_functions::RandomProjection) { - out << "random_projections"; - return out; + return out << "random_projections"; +} + + +/** + * @brief Struct to encapsulate all possible hash functions probing tags. + */ +struct probing { + /** + * @brief Tag for the LSH using single (normal) probing. + */ + static struct Single{} single; + /** + * @brief Tag for the LSH using multi-probing. + */ + static struct Multiple{} multiple; +}; +/** + * @brief Stream insertion operator overload for the @ref probing::Single tag. + * @param[in] out the output stream + * @return the output stream + */ +std::ostream& operator<<(std::ostream& out, probing::Single) { + return out << "single"; +} +/** + * @brief Stream insertion operator overload for the @ref probing::Multiple tag. + * @param[in] out the output stream + * @return the output stream + */ +std::ostream& operator<<(std::ostream& out, probing::Multiple) { + return out << "multiple"; } diff --git a/include/hash_table.hpp b/include/hash_table.hpp index 572cf91..a91aa06 100644 --- a/include/hash_table.hpp +++ b/include/hash_table.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-08-04 * * @brief Implements the @ref hash_tables class representing the used LSH hash tables. */ @@ -20,6 +20,14 @@ #include +struct pair_type { + pair_type() = default; + pair_type(const std::uint32_t idx, const std::int32_t delta) : idx(idx), delta(delta) { } + + std::uint32_t idx = 0; + std::int32_t delta = 0.0; +}; + /** * @brief Class representing the hash tables used in the LSH algorithm. * @tparam layout determines whether the hash functions are saved as *Array of Structs* or *Struct of Arrays* @@ -44,7 +52,53 @@ class hash_tables : detail::hash_tables_base { /// The SYCL buffer holding the hash bucket offsets: `offsets.get_count() == options::num_hash_tables * (options::hash_table_size + 1)`. sycl::buffer offsets; /// Hash functions used by this hash tables. - HashFunctions hash_function; + HashFunctions hash_functions; + + + template + void execute(const int comm_rank, const bool first_round, index_type* nearest_neighbors, real_type* distances, real_type& max_distance, index_type& argmax, + const index_type k, const index_type hash_table, const hash_value_type hash_bucket, const index_type idx, + AccOffsets& acc_offsets, AccHashTables& acc_hash_tables, AccDataReceived& acc_data_received, AccDataOwned& acc_data_owned, + const Data& data, const Options& opt) + { + for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; + bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; + ++bucket_element) + { + const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; + const index_type point_idx = point % data.rank_size; + real_type dist = 0.0; + for (index_type dim = 0; dim < data.dims; ++dim) { + const index_type x_idx = data.get_linear_id(comm_rank, idx, data.rank_size, dim, data.dims); + const real_type x = acc_data_received[x_idx]; + const index_type y_idx = data.get_linear_id(comm_rank, point_idx, data.rank_size, dim, data.dims); + const real_type y = acc_data_owned[y_idx]; + + dist += (x - y) * (x - y); + } + + // updated nearest-neighbors + const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { + if (first_round && point_idx == idx) return false; + for (index_type i = 0; i < k; ++i) { + if (neighbors[i] == point) return false; + } + return true; + }; + if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { + nearest_neighbors[argmax] = point; + distances[argmax] = dist; + max_distance = dist; + for (index_type i = 0; i < k; ++i) { + if (distances[i] > max_distance) { + max_distance = distances[i]; + argmax = i; + } + } + } + } + } + template void calculate_knn(const index_type k, Knns& knns) { @@ -83,7 +137,7 @@ class hash_tables : detail::hash_tables_base { auto acc_data_owned = data_.buffer.template get_access(cgh); auto acc_data_received = data_buffer.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_offsets = offsets.template get_access(cgh); auto acc_hash_tables = buffer.template get_access(cgh); auto acc_knns = knn_buffers.template get_access(cgh); @@ -96,8 +150,15 @@ class hash_tables : detail::hash_tables_base { if (idx >= data.rank_size) return; - index_type* nearest_neighbors = new index_type[k]; - real_type* distances = new real_type[k]; + auto* nearest_neighbors = new index_type[k]; + auto* distances = new real_type[k]; + + pair_type* probes = nullptr; + real_type* probe_distances = nullptr; + if constexpr (std::is_same_v, probing::Multiple>) { + probes = new pair_type[opt.num_multi_probes]; + probe_distances = new real_type[opt.num_multi_probes]; + } // initialize arrays for (index_type i = 0; i < k; ++i) { @@ -114,42 +175,50 @@ class hash_tables : detail::hash_tables_base { } for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - const hash_value_type hash_bucket = hash_function.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data); - - for (index_type bucket_element = acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket]; - bucket_element < acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_bucket + 1]; - ++bucket_element) - { - const index_type point = acc_hash_tables[hash_table * data.rank_size + bucket_element]; - const index_type point_idx = point % data.rank_size; - real_type dist = 0.0; - for (index_type dim = 0; dim < data.dims; ++dim) { - const index_type x_idx = data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims); - const real_type x = acc_data_received[x_idx]; - const index_type y_idx = data.get_linear_id(comm_rank_, point_idx, data.rank_size, dim, data.dims); - const real_type y = acc_data_owned[y_idx]; - - dist += (x - y) * (x - y); + const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, acc_hash_functions, opt, data); + execute(comm_rank_, first_round, nearest_neighbors, distances, max_distance, argmax, k, hash_table, hash_bucket, idx, + acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned, data, opt); + + if constexpr (std::is_same_v, probing::Multiple>) { + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + probe_distances[i] = std::numeric_limits::max(); } - // updated nearest-neighbors - const auto is_candidate = [=](const auto point, const index_type* neighbors, const index_type k, const index_type idx) { - if (first_round && point_idx == idx) return false; - for (index_type i = 0; i < k; ++i) { - if (neighbors[i] == point) return false; + index_type probe_argmax = 0; + real_type probe_max = probe_distances[probe_argmax]; + + const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hf, const std::uint32_t delta) { + if (probe_max > slot_dist) { + probes[probe_argmax] = pair_type(hf, delta); + probe_distances[probe_argmax] = slot_dist; + probe_max = slot_dist; + for (index_type i = 0; i < opt.num_multi_probes; ++i) { + if (probe_distances[i] > probe_max) { + probe_max = probe_distances[i]; + probe_argmax = i; + } + } } - return true; }; - if (dist < max_distance && is_candidate(point, nearest_neighbors, k, idx)) { - nearest_neighbors[argmax] = point; - distances[argmax] = dist; - max_distance = dist; - for (index_type i = 0; i < k; ++i) { - if (distances[i] > max_distance) { - max_distance = distances[i]; - argmax = i; - } + + // calculate distances + for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { + real_type hash = acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, data.dims, opt, data)]; + for (index_type dim = 0; dim < data.dims; ++dim) { + hash += acc_data_received[data.get_linear_id(comm_rank_, idx, data.rank_size, dim, data.dims)] * + acc_hash_functions[hash_functions.get_linear_id(comm_rank_, hash_table, hash_function, dim, opt, data)]; } + + const auto slot = static_cast(hash / opt.w); + update_probing_sequence(hash - slot * opt.w, hash_function, -1); + update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); + } + + for (index_type probe = 0; probe < opt.num_multi_probes; ++probe) { + const hash_value_type hash_bucket = hash_functions.hash(comm_rank_, hash_table, idx, acc_data_received, + acc_hash_functions, opt, data, probes[probe].idx, probes[probe].delta); + execute(comm_rank_, first_round, nearest_neighbors, distances, max_distance, argmax, k, hash_table, hash_bucket, idx, + acc_offsets, acc_hash_tables, acc_data_received, acc_data_owned, data, opt); } } } @@ -162,6 +231,8 @@ class hash_tables : detail::hash_tables_base { delete[] nearest_neighbors; delete[] distances; + delete[] probes; + delete[] probe_distances; }); }); END_TIMING_MPI_AND_BARRIER(calculate_nearest_neighbors, comm_rank_, queue_); @@ -207,7 +278,7 @@ class hash_tables : detail::hash_tables_base { */ hash_tables(sycl::queue& queue, const Options& opt, Data& data, HashFunctions hash_functions, const int comm_rank) : buffer(opt.num_hash_tables * data.rank_size), offsets(opt.num_hash_tables * (opt.hash_table_size + 1)), - hash_function(hash_functions), queue_(queue), comm_rank_(comm_rank), opt_(opt), data_(data) + hash_functions(hash_functions), queue_(queue), comm_rank_(comm_rank), opt_(opt), data_(data) { { // create temporary buffer to count the occurrence of each hash value @@ -243,7 +314,7 @@ class hash_tables : detail::hash_tables_base { START_TIMING(count_hash_values); queue_.submit([&](sycl::handler& cgh) { auto acc_hash_value_count = hash_value_count.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_data = data_.buffer.template get_access(cgh); auto opt = opt_; auto data = data_; @@ -256,7 +327,7 @@ class hash_tables : detail::hash_tables_base { for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { const hash_value_type hash_value = - hash_function.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); + hash_functions.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); acc_hash_value_count[hash_table * opt.hash_table_size + hash_value].fetch_add(1); } }); @@ -300,7 +371,7 @@ class hash_tables : detail::hash_tables_base { START_TIMING(fill_hash_tables); queue_.submit([&](sycl::handler& cgh) { auto acc_data = data_.buffer.template get_access(cgh); - auto acc_hash_functions = hash_function.buffer.template get_access(cgh); + auto acc_hash_functions = hash_functions.buffer.template get_access(cgh); auto acc_offsets = offsets.template get_access(cgh); auto acc_hash_tables = buffer.template get_access(cgh); auto opt = opt_; @@ -311,7 +382,7 @@ class hash_tables : detail::hash_tables_base { const index_type idx = item.get_linear_id(); for (index_type hash_table = 0; hash_table < opt.num_hash_tables; ++hash_table) { - const hash_value_type hash_value = hash_function.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); + const hash_value_type hash_value = hash_functions.hash(comm_rank, hash_table, idx, acc_data, acc_hash_functions, opt, data); acc_hash_tables[hash_table * data.rank_size + acc_offsets[hash_table * (opt.hash_table_size + 1) + hash_value + 1].fetch_add(1)] = idx + comm_rank * data.rank_size; } diff --git a/include/options.hpp b/include/options.hpp index 06a3a57..a485c61 100644 --- a/include/options.hpp +++ b/include/options.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-07-31 * * @brief Implements a @ref options class for managing hyperparameters. */ @@ -21,7 +21,6 @@ #include #include -#include #include #include @@ -92,14 +91,18 @@ struct options : detail::options_base { this->set_hash_table_size(detail::convert_to(value)); } else if (opt == "num_hash_functions") { this->set_num_hash_functions(detail::convert_to(value)); + } else if (opt == "num_multi_probes") { + this->set_num_multi_probes(detail::convert_to(value)); } else if (opt == "w") { this->set_w(detail::convert_to(value)); - } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" || opt == "hash_functions_type") { + } else if (opt == "real_type" || opt == "index_type" || opt == "hash_value_type" + || opt == "hash_functions_type" || opt == "probing_type") + { in.ignore(std::numeric_limits::max(), '\n'); } else { - const std::string msg = std::string("Invalid option '").append(opt).append(" ") - .append(value).append("' in file '").append(file).append("'."); - throw std::invalid_argument(msg); + std::stringstream ss; + ss << "Invalid option '" << opt << " " << "' in file '" << file << "'."; + throw std::invalid_argument(ss.str()); } } } @@ -110,9 +113,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_hash_pool_size **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_hash_pool_size isn't greater than `0` */ factory& set_hash_pool_size(const index_type factory_hash_pool_size) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_hash_pool_size, "Illegal number of hash functions in hash pool!: 0 < {}", factory_hash_pool_size); + if (factory_hash_pool_size <= 0) { + throw std::invalid_argument("The number of hash functions in the hash pool must be greater than 0!"); + } hash_pool_size_ = factory_hash_pool_size; return *this; } @@ -120,9 +127,15 @@ struct options : detail::options_base { * @brief Set the new number of cut-of points. * @param[in] factory_num_cut_off_points the number of cut-off points used in the entropy-based hash functions * @return `*this` + * + * @pre @p factory_num_cut_off_points **must** be greater than `0` + * + * @throws std::invalid_argument if @p factory_num_cut_off_points isn't greater than `0` */ factory& set_num_cut_off_points(const index_type factory_num_cut_off_points) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_cut_off_points, "Illegal number of cut-off points!: 0 < {}", factory_num_cut_off_points); + if (factory_num_cut_off_points <= 0) { + throw std::invalid_argument("The number of cut-off points used by the entropy-based hash functions must be greater than 0!"); + } num_cut_off_points_ = factory_num_cut_off_points; return *this; } @@ -132,9 +145,13 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_num_hash_tables **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_hash_tables isn't greater than `0` */ factory& set_num_hash_tables(const index_type factory_num_hash_tables) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_hash_tables, "Illegal number of hash tables!: 0 < {}", factory_num_hash_tables); + if (factory_num_hash_tables <= 0) { + throw std::invalid_argument("The number of hash tables must be greater than 0!"); + } num_hash_tables_ = factory_num_hash_tables; return *this; } @@ -146,10 +163,16 @@ struct options : detail::options_base { * * @pre @p factory_hash_table_size **must** be greater than `0`. * @pre @p factory_hash_table_size **must** be a prime number. + * + * @throws std::invalid_argument if @p factory_hash_table_size isn't greater than `0` or isn't a prime number */ factory& set_hash_table_size(const hash_value_type factory_hash_table_size) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_hash_table_size, "Illegal hash_table_size!: 0 < {}", factory_hash_table_size); - DEBUG_ASSERT_MPI(comm_rank_, this->is_prime(factory_hash_table_size), "{} is not a prime!", factory_hash_table_size); + if (factory_hash_table_size <= 0) { + throw std::invalid_argument("The size of a hash table must be greater than 0!"); + } + if (!this->is_prime(factory_hash_table_size)) { + throw std::invalid_argument("The size of a hash table must be a prime number!"); + } hash_table_size_ = factory_hash_table_size; return *this; } @@ -159,21 +182,45 @@ struct options : detail::options_base { * @return `*this` * * @pre @p factory_num_hash_functions **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_hash_functions isn't greater than `0` */ factory& set_num_hash_functions(const index_type factory_num_hash_functions) { - DEBUG_ASSERT_MPI(comm_rank_, 0 < factory_num_hash_functions, "Illegal number of hash functions!: 0 < {}", factory_num_hash_functions); + if (factory_num_hash_functions <= 0) { + throw std::invalid_argument("The number of hash functions per hash table must be greater than 0!"); + } num_hash_functions_ = factory_num_hash_functions; return *this; } + /** + * @brief Set the new number of multi-probes to perform per hash table. + * @param[in] factory_num_multi_probes number of multi-probes + * @return `*this` + * + * @pre @p factory_num_multi_probes **must** be greater than `0`. + * + * @throws std::invalid_argument if @p factory_num_multi_probes isn't greater than `0` + */ + factory& set_num_multi_probes(const index_type factory_num_multi_probes) { + if (factory_num_multi_probes <= 0) { + throw std::invalid_argument("The number of multi-probes per hash table must be greater than 0!"); + } + num_multi_probes_ = factory_num_multi_probes; + return *this; + } /** * @brief Set the new w value used in the hash value calculation: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. * @param[in] factory_w constant value for the hash value calculation * @return `*this` * * @pre @p factory_w **must** be greater than `0.0`. + * + * @throws std::invalid_argument if @p factory_w isn't greater than `0.0` */ factory& set_w(const real_type factory_w) { - DEBUG_ASSERT_MPI(comm_rank_, 0.0 < factory_w, "Illegal 'w' value!: 0.0 < {}", factory_w); + if (factory_w <= 0.0) { + throw std::invalid_argument("The value of w must be greater than 0.0!"); + } w_ = factory_w; return *this; } @@ -209,6 +256,7 @@ struct options : detail::options_base { index_type num_hash_tables_ = static_cast(2); hash_value_type hash_table_size_ = static_cast(105613); index_type num_hash_functions_ = static_cast(4); + index_type num_multi_probes_ = static_cast(6); real_type w_ = static_cast(1.0); int comm_rank_; @@ -222,7 +270,15 @@ struct options : detail::options_base { options(options::factory fact) : hash_pool_size(fact.hash_pool_size_), num_cut_off_points(fact.num_cut_off_points_), num_hash_tables(fact.num_hash_tables_), hash_table_size(fact.hash_table_size_), - num_hash_functions(fact.num_hash_functions_), w(fact.w_) { } + num_hash_functions(fact.num_hash_functions_), num_multi_probes(fact.num_multi_probes_), w(fact.w_) + { + // TODO 2020-07-30 16:48 marcel: meaningful restriction? + if constexpr (std::is_same_v) { + if (num_multi_probes > num_hash_functions) { + throw std::invalid_argument("The number of multi-probes can't be greater than the number of hash functions!"); + } + } + } // ---------------------------------------------------------------------------------------------------------- // // runtime options // @@ -238,6 +294,8 @@ struct options : detail::options_base { const hash_value_type hash_table_size; /// The number of hash functions per hash table. const index_type num_hash_functions; + /// The number of multi-probes to perform per hash table. + const index_type num_multi_probes; /// A constant used in the hash functions: \f$h_{a, b} = \frac{a \cdot x + b}{w}\f$. const real_type w; @@ -247,6 +305,8 @@ struct options : detail::options_base { /// The type of the hash functions. static constexpr auto hash_functions_type = hash_functions::random_projection; + /// Specifies whether multi-probe LSH should be used. + static constexpr auto probing_type = probing::single; /** @@ -276,11 +336,13 @@ struct options : detail::options_base { out << "index_type '" << boost::typeindex::type_id().pretty_name() << "'\n"; out << "hash_value_type '" << boost::typeindex::type_id().pretty_name() << "'\n"; out << "hash_functions_type '" << options::hash_functions_type << "'\n"; + out << "probing_type '" << options::probing_type << "'\n"; out << "hash_pool_size " << opt.hash_pool_size << '\n'; out << "num_cut_off_points " << opt.num_cut_off_points << '\n'; out << "num_hash_tables " << opt.num_hash_tables << '\n'; out << "hash_table_size " << opt.hash_table_size << '\n'; out << "num_hash_functions " << opt.num_hash_functions << '\n'; + out << "num_multi_probes " << opt.num_multi_probes << '\n'; out << "w " << opt.w; return out; diff --git a/include/random_projection_hash_function.hpp b/include/random_projection_hash_function.hpp index a01b5ef..2704457 100644 --- a/include/random_projection_hash_function.hpp +++ b/include/random_projection_hash_function.hpp @@ -1,7 +1,7 @@ /** * @file * @author Marcel Breyer - * @date 2020-07-29 + * @date 2020-08-04 * * @brief Implements the @ref random_projection_hash_functions class representing the used LSH hash functions. */ @@ -68,7 +68,7 @@ class random_projection_hash_functions : detail::hash_functions_base { [[nodiscard]] static constexpr hash_value_type hash([[maybe_unused]] const int comm_rank, const index_type hash_table, const index_type point, AccData& acc_data, AccHashFunctions& acc_hash_functions, - const Options& opt, const Data& data) + const Options& opt, const Data& data, const index_type idx = 0, const int delta = 0) { DEBUG_ASSERT_MPI(comm_rank, 0 <= hash_table && hash_table < opt.num_hash_tables, "Out-of-bounce access!: 0 <= {} < {}", hash_table, opt.num_hash_tables); @@ -82,7 +82,8 @@ class random_projection_hash_functions : detail::hash_functions_base { hash += acc_data[data_type::get_linear_id(comm_rank, point, data.rank_size, dim, data.dims)] * acc_hash_functions[get_linear_id(comm_rank, hash_table, hash_function, dim, opt, data)]; } - combined_hash ^= static_cast(hash / opt.w) + auto slot = static_cast(hash / opt.w); + combined_hash ^= (hash_function == idx ? slot + delta : slot) + static_cast(0x9e3779b9) + (combined_hash << static_cast(6)) + (combined_hash >> static_cast(2)); @@ -177,7 +178,6 @@ class random_projection_hash_functions : detail::hash_functions_base { template friend class random_projection_hash_functions; - /** * @brief Construct new hash functions. * @param[in] opt the @ref options object representing the currently set options @@ -192,17 +192,66 @@ class random_projection_hash_functions : detail::hash_functions_base { for (std::size_t i = 0; i < tmp_buffer.size(); ++i) { acc[i] = tmp_buffer[i]; } +// +// sycl::queue queue(sycl::default_selector{}); +// queue.submit([&](sycl::handler& cgh) { +// auto acc_probing_sequence = probing_sequence.template get_access(cgh); +// auto acc_data = data.buffer.template get_access(cgh); +// auto acc_hash_functions = data.buffer.template get_access(cgh); +// auto data = data_; +// auto opt = opt_; +// auto comm_rank = comm_rank_; +// +// cgh.parallel_for(sycl::range<>(data.rank_size), [=](sycl::item<> item) { +// const index_type idx = item.get_linear_id(); +// +// auto* probes = new pair_type[opt.num_multi_probes]; +// auto* dists = new real_type[opt.num_multi_probes]; +// +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// dists[i] = std::numeric_limits::max(); +// } +// +// index_type argmax = 0; +// real_type max = dists[argmax]; +// +// const auto update_probing_sequence = [&](const real_type slot_dist, const index_type hash_function, const signed_type delta) { +// if (max > slot_dist) { +// probes[argmax] = pair_type(hash_function, delta); +// dists[argmax] = slot_dist; +// max = slot_dist; +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// if (dists[i] > max) { +// max = dists[i]; +// argmax = i; +// } +// } +// } +// }; +// +// // calculate distances +// for (index_type hash_function = 0; hash_function < opt.num_hash_functions; ++hash_function) { +// real_type hash = acc_hash_functions[get_linear_id(comm_rank, 0, hash_function, data.dims, opt, data)]; +// for (index_type dim = 0; dim < data.dims; ++dim) { +// hash += acc_data[data_type::get_linear_id(comm_rank, idx, data.rank_size, dim, data.dims)] * +// acc_hash_functions[get_linear_id(comm_rank, 0, hash_function, dim, opt, data)]; +// } +// +// const auto slot = static_cast(hash / opt.w); +// update_probing_sequence(hash - slot * opt.w, hash_function, -1); +// update_probing_sequence(opt.w - (hash - slot * opt.w), hash_function, +1); +// } +// +// for (index_type i = 0; i < opt.num_multi_probes; ++i) { +// acc_probing_sequence[idx * opt.num_multi_probes + i] = probes[i]; +// } +// +// delete[] probes; +// delete[] dists; +// }); +// }); } - /** - * @brief Construct an empty hash functions buffer. - * @param[in] opt the @ref options object representing the currently set options - * @param[in] data the @ref data object representing the used data set - * @param[in] size the size of the empty buffer - * @param[in] comm_rank the current MPI rank - */ - random_projection_hash_functions(const Options& opt, Data& data, const index_type size, const int comm_rank) - : buffer(size), comm_rank_(comm_rank), opt_(opt), data_(data) { } /// The current MPI rank. const int comm_rank_; diff --git a/src/main.cpp b/src/main.cpp index f93327c..3046bf7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -174,6 +174,10 @@ int custom_main(MPI_Comm& communicator, const int argc, char** argv) { options_factory.set_num_hash_functions( parser.argv_as().num_hash_functions)>>("num_hash_functions")); } + if (parser.has_argv("num_multi_probes")) { + options_factory.set_num_multi_probes( + parser.argv_as().num_multi_probes)>>("num_multi_probes")); + } if (parser.has_argv("w")) { options_factory.set_w( parser.argv_as().w)>>("w"));