Skip to content

Commit 0577e45

Browse files
jotojoto
committed
Introduce new "id cache" for node tables
It is sometimes useful to know whether a way contains nodes of a certain type, i.e. with certain tags. This commit adds a new functionality called "id caches" that allows to find out. When defining a node table in Lua with `define_table` the `ids` section can now contain a field `cache = true`. If this is set the ids of all nodes written to that table are stored in memory. Later, when processing ways, that cache can be queried with `tablename:in_id_cache()`. Usually this would be used to query all the member nodes of a way: `...:in_id_cache(object.nodes)`. The `in_id_cache()` function returns the indexes into the `object.nodes` array where the id matches. From there it is possible to get the id of the matching node (with `object.nodes[idx]`) or the location of that node (with `object:as_point(idx)`. This information can be stored in a way table which will be correctly updated in append mode. (In append mode the in-memory id cache is populated with all the ids of the table for which the cache was defined.) The whole thing only works for node tables and querying only works in the process_way() function. It can possibly be extended later to work in other contexts.
1 parent 5464539 commit 0577e45

File tree

11 files changed

+648
-0
lines changed

11 files changed

+648
-0
lines changed

‎flex-config/turning-circles.lua‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
-- This config example file is released into the Public Domain.
2+
3+
-- Create a table with turning circles that can be styled in sync with the
4+
-- highway they are on.
5+
6+
local turning_circles = osm2pgsql.define_table({
7+
name = 'turning_circles',
8+
ids = { type = 'node', id_column = 'node_id', cache = true },
9+
columns = {
10+
{ column = 'geom', type = 'point', not_null = true },
11+
}
12+
})
13+
14+
local highways = osm2pgsql.define_table({
15+
name = 'highways',
16+
ids = { type = 'way', id_column = 'way_id' },
17+
columns = {
18+
{ column = 'htype', type = 'text', not_null = true },
19+
{ column = 'geom', type = 'linestring', not_null = true },
20+
}
21+
})
22+
23+
-- This table will contain entries for all node/way combinations where the way
24+
-- is tagged as "highway" and the node is tagged as "highway=turning_circle".
25+
-- The "htype" column contains the highway type, the "geom" the geometry of
26+
-- the node. This can be used, for instance, to draw the point in a style that
27+
-- fits with the style of the highway.
28+
--
29+
-- Note that you might have multiple entries for the same node in this table
30+
-- if it is in several ways. In that case you might have to decide at rendering
31+
-- time which of them to render.
32+
local highway_ends = osm2pgsql.define_table({
33+
name = 'highway_ends',
34+
ids = { type = 'way', id_column = 'way_id' },
35+
columns = {
36+
{ column = 'htype', type = 'text', not_null = true },
37+
{ column = 'node_id', type = 'int8', not_null = true },
38+
{ column = 'geom', type = 'point', not_null = true },
39+
}
40+
})
41+
42+
function osm2pgsql.process_node(object)
43+
if object.tags.highway == 'turning_circle' then
44+
-- This insert will add the entry to the id cache later read with
45+
-- in_id_cache().
46+
turning_circles:insert({
47+
geom = object:as_point(),
48+
})
49+
end
50+
end
51+
52+
function osm2pgsql.process_way(object)
53+
local t = object.tags.highway
54+
if t then
55+
highways:insert({
56+
htype = t,
57+
geom = object:as_linestring(),
58+
})
59+
local c = turning_circles:in_id_cache(object.nodes)
60+
for _, n in ipairs(c) do
61+
highway_ends:insert({
62+
htype = t,
63+
node_id = object.nodes[n],
64+
geom = object:as_point(n),
65+
})
66+
end
67+
end
68+
end
69+

‎src/debug-output.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ void write_table_list_to_debug_log(std::vector<flex_table_t> const &tables)
6161
log_debug(" - data_tablespace={}", table.data_tablespace());
6262
log_debug(" - index_tablespace={}", table.index_tablespace());
6363
log_debug(" - cluster={}", table.cluster_by_geom());
64+
log_debug(" - id_cache={}", table.with_id_cache());
6465
for (auto const &index : table.indexes()) {
6566
log_debug(" - INDEX USING {}", index.method());
6667
if (index.name().empty()) {

‎src/flex-lua-table.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,17 @@ void setup_flex_table_id_columns(lua_State *lua_state, flex_table_t *table)
174174
throw fmt_error("Unknown ids type: {}.", type);
175175
}
176176

177+
bool const cache =
178+
luaX_get_table_bool(lua_state, "cache", -1, "The ids", false);
179+
lua_pop(lua_state, 1); // "cache"
180+
if (cache) {
181+
if (type == "node") {
182+
table->enable_id_cache();
183+
} else {
184+
throw std::runtime_error{"ID cache only available for node ids."};
185+
}
186+
}
187+
177188
std::string const name =
178189
luaX_get_table_string(lua_state, "id_column", -1, "The ids field");
179190
lua_pop(lua_state, 1); // "id_column"
@@ -459,6 +470,7 @@ void lua_wrapper_table_t::init(lua_State *lua_state)
459470
luaX_set_up_metatable(lua_state, "Table", OSM2PGSQL_TABLE_CLASS,
460471
{{"__tostring", lua_trampoline_table_tostring},
461472
{"insert", lua_trampoline_table_insert},
473+
{"in_id_cache", lua_trampoline_table_in_id_cache},
462474
{"name", lua_trampoline_table_name},
463475
{"schema", lua_trampoline_table_schema},
464476
{"cluster", lua_trampoline_table_cluster},

‎src/flex-table.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,10 @@ void flex_table_t::analyze(pg_conn_t const &db_connection) const
263263
analyze_table(db_connection, schema(), name());
264264
}
265265

266+
void flex_table_t::enable_id_cache() noexcept { m_with_id_cache = true; }
267+
268+
bool flex_table_t::with_id_cache() const noexcept { return m_with_id_cache; }
269+
266270
namespace {
267271

268272
void enable_check_trigger(pg_conn_t const &db_connection,

‎src/flex-table.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,10 @@ class flex_table_t
215215

216216
void analyze(pg_conn_t const &db_connection) const;
217217

218+
void enable_id_cache() noexcept;
219+
220+
bool with_id_cache() const noexcept;
221+
218222
private:
219223
/// The schema this table is in
220224
std::string m_schema;
@@ -271,6 +275,9 @@ class flex_table_t
271275
/// Index should be a primary key.
272276
bool m_primary_key_index = false;
273277

278+
/// Do we want an ID cache for this table?
279+
bool m_with_id_cache = false;
280+
274281
}; // class flex_table_t
275282

276283
class table_connection_t

‎src/idlist.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ osmid_t idlist_t::pop_id()
2222
return id;
2323
}
2424

25+
bool idlist_t::contains(osmid_t id) const
26+
{
27+
return std::binary_search(m_list.begin(), m_list.end(), id);
28+
}
29+
2530
void idlist_t::sort_unique()
2631
{
2732
std::sort(m_list.begin(), m_list.end());

‎src/idlist.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,13 @@ class idlist_t
6262

6363
void reserve(std::size_t size) { m_list.reserve(size); }
6464

65+
/**
66+
* Is the specified id in the list?
67+
*
68+
* You must have called sort_unique() before calling this.
69+
*/
70+
bool contains(osmid_t id) const;
71+
6572
/**
6673
* Remove id at the end of the list and return it.
6774
*

‎src/output-flex.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ TRAMPOLINE(app_as_geometrycollection, as_geometrycollection)
8989
} // anonymous namespace
9090

9191
TRAMPOLINE(table_insert, insert)
92+
TRAMPOLINE(table_in_id_cache, in_id_cache)
9293

9394
prepared_lua_function_t::prepared_lua_function_t(lua_State *lua_state,
9495
calling_context context,
@@ -270,6 +271,9 @@ void flush_tables(std::vector<table_connection_t> &table_connections)
270271
for (auto &table : table_connections) {
271272
table.flush();
272273
}
274+
for (auto &table : table_connections) {
275+
table.sync();
276+
}
273277
}
274278

275279
void create_expire_tables(std::vector<expire_output_t> const &expire_outputs,
@@ -789,6 +793,10 @@ int output_flex_t::table_insert()
789793
auto const &object = check_and_get_context_object(table);
790794
osmid_t const id = table.map_id(object.type(), object.id());
791795

796+
if (table.with_id_cache()) {
797+
get_id_cache(table).push_back(id);
798+
}
799+
792800
table_connection.new_line();
793801
auto *copy_mgr = table_connection.copy_mgr();
794802

@@ -823,6 +831,46 @@ int output_flex_t::table_insert()
823831
return 1;
824832
}
825833

834+
int output_flex_t::table_in_id_cache()
835+
{
836+
if (m_calling_context == calling_context::process_node) {
837+
throw std::runtime_error{
838+
"Id cache not available while processing nodes."};
839+
}
840+
841+
// The first parameter is the table object.
842+
auto const &table = get_table_from_param();
843+
844+
if (!table.with_id_cache()) {
845+
throw fmt_error("No ID cache on table '{}'.", table.name());
846+
}
847+
848+
// The second parameter is an array of ids
849+
if (!lua_istable(lua_state(), 1) || !luaX_is_array(lua_state())) {
850+
throw std::runtime_error{"Second parameter must be an array of ids."};
851+
}
852+
853+
std::vector<osmid_t> ids;
854+
luaX_for_each(lua_state(),
855+
[&]() { ids.push_back(lua_tointeger(lua_state(), -1)); });
856+
857+
auto const &cache = get_id_cache(table);
858+
lua_createtable(lua_state(), 0, 0);
859+
860+
lua_Integer n = 0;
861+
lua_Integer idx = 1;
862+
for (auto const id : ids) {
863+
if (cache.contains(id)) {
864+
lua_pushinteger(lua_state(), ++n);
865+
lua_pushinteger(lua_state(), idx);
866+
lua_rawset(lua_state(), -3);
867+
}
868+
++idx;
869+
}
870+
871+
return 1;
872+
}
873+
826874
void output_flex_t::call_lua_function(prepared_lua_function_t func)
827875
{
828876
lua_pushvalue(lua_state(), func.index());
@@ -980,6 +1028,28 @@ void output_flex_t::after_nodes()
9801028
}
9811029

9821030
flush_tables(m_table_connections);
1031+
1032+
for (auto &table : *m_tables) {
1033+
if (table.with_id_cache()) {
1034+
auto &cache = get_id_cache(table);
1035+
if (get_options()->append) {
1036+
log_debug("Initializing cache for table '{}' from database...",
1037+
table.name());
1038+
auto const result = m_db_connection.exec(
1039+
"SELECT \"{}\" FROM {}", table.id_column_names(),
1040+
table.full_name());
1041+
1042+
cache.reserve(result.num_tuples());
1043+
for (int i = 0; i < result.num_tuples(); ++i) {
1044+
cache.push_back(
1045+
osmium::string_to_object_id(result.get_value(i, 0)));
1046+
}
1047+
}
1048+
cache.sort_unique();
1049+
log_debug("Cache for table '{}' initialized with {} entries.",
1050+
table.name(), cache.size());
1051+
}
1052+
}
9831053
}
9841054

9851055
void output_flex_t::after_ways()
@@ -1199,6 +1269,10 @@ void output_flex_t::relation_modify(osmium::Relation const &rel)
11991269
void output_flex_t::start()
12001270
{
12011271
for (auto &table : m_table_connections) {
1272+
if (table.table().with_id_cache()) {
1273+
log_debug("Enable cache for table '{}'.", table.table().name());
1274+
create_id_cache(table.table());
1275+
}
12021276
table.start(m_db_connection, get_options()->append);
12031277
}
12041278

@@ -1212,6 +1286,7 @@ output_flex_t::output_flex_t(output_flex_t const *other,
12121286
std::shared_ptr<db_copy_thread_t> copy_thread)
12131287
: output_t(other, std::move(mid)), m_locators(other->m_locators),
12141288
m_tables(other->m_tables), m_expire_outputs(other->m_expire_outputs),
1289+
m_id_caches(other->m_id_caches),
12151290
m_db_connection(get_options()->connection_params, "out.flex.thread"),
12161291
m_stage2_way_ids(other->m_stage2_way_ids),
12171292
m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state),

‎src/output-flex.hpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ class output_flex_t : public output_t
165165
int app_get_bbox();
166166

167167
int table_insert();
168+
int table_in_id_cache();
168169

169170
// Get the flex table that is as first parameter on the Lua stack.
170171
flex_table_t &get_table_from_param();
@@ -216,6 +217,21 @@ class output_flex_t : public output_t
216217

217218
lua_State *lua_state() noexcept { return m_lua_state.get(); }
218219

220+
void create_id_cache(flex_table_t const &table)
221+
{
222+
if (table.num() >= m_id_caches.size()) {
223+
m_id_caches.resize(table.num() + 1);
224+
}
225+
m_id_caches[table.num()] = std::make_shared<idlist_t>();
226+
}
227+
228+
idlist_t &get_id_cache(flex_table_t const &table)
229+
{
230+
auto& c = m_id_caches[table.num()];
231+
assert(c);
232+
return *c;
233+
}
234+
219235
class way_cache_t
220236
{
221237
public:
@@ -273,6 +289,8 @@ class output_flex_t : public output_t
273289
std::shared_ptr<std::vector<expire_output_t>> m_expire_outputs =
274290
std::make_shared<std::vector<expire_output_t>>();
275291

292+
std::vector<std::shared_ptr<idlist_t>> m_id_caches;
293+
276294
std::vector<table_connection_t> m_table_connections;
277295

278296
/// The connection to the database server.
@@ -325,5 +343,6 @@ class output_flex_t : public output_t
325343
};
326344

327345
int lua_trampoline_table_insert(lua_State *lua_state);
346+
int lua_trampoline_table_in_id_cache(lua_State *lua_state);
328347

329348
#endif // OSM2PGSQL_OUTPUT_FLEX_HPP

0 commit comments

Comments
 (0)