From 65dee52179b918a83248aa88549c762cd299a12c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 22 Sep 2025 09:58:19 +0200 Subject: [PATCH 01/45] install osm2pgsql-expire --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22d37fb5b..968fc797c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -330,6 +330,7 @@ include(GNUInstallDirs) if (ENABLE_INSTALL) install(TARGETS osm2pgsql DESTINATION bin) + install(TARGETS osm2pgsql-expire DESTINATION bin) install(FILES default.style empty.style DESTINATION share/osm2pgsql) install(PROGRAMS scripts/osm2pgsql-replication DESTINATION bin) if (BUILD_GEN) From c98b02af0401b27c1ef762e872c4427d1d3bee49 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 24 Oct 2025 11:45:21 +0200 Subject: [PATCH 02/45] Implement expire of points in its own code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of generating a bounding box and using the general bbox expire code. This takes the expire buffer into account, so neighboring tiles are expired if the point is near the tile boundary. Other than before tiles affected wrap around at the 180° line. --- src/expire-tiles.cpp | 18 ++++++++++++++++-- tests/test-expire-tiles.cpp | 6 ++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/expire-tiles.cpp b/src/expire-tiles.cpp index d9578cd89..8ad2e6c63 100644 --- a/src/expire-tiles.cpp +++ b/src/expire-tiles.cpp @@ -71,8 +71,22 @@ void expire_tiles_t::from_point_list(geom::point_list_t const &list, void expire_tiles_t::from_geometry(geom::point_t const &geom, expire_config_t const &expire_config) { - geom::box_t const box = geom::envelope(geom); - from_bbox(box, expire_config); + auto const tilec = coords_to_tile(geom); + + auto const ymin = + std::max(0U, static_cast(tilec.y() - expire_config.buffer)); + + auto const ymax = + std::min(m_map_width - 1U, + static_cast(tilec.y() + expire_config.buffer)); + + for (int x = static_cast(tilec.x() - expire_config.buffer); + x <= static_cast(tilec.x() + expire_config.buffer); ++x) { + uint32_t const norm_x = normalise_tile_x_coord(x); + for (uint32_t y = ymin; y <= ymax; ++y) { + expire_tile(norm_x, y); + } + } } void expire_tiles_t::from_geometry(geom::linestring_t const &geom, diff --git a/tests/test-expire-tiles.cpp b/tests/test-expire-tiles.cpp index 844bc49eb..b8409202d 100644 --- a/tests/test-expire-tiles.cpp +++ b/tests/test-expire-tiles.cpp @@ -189,9 +189,10 @@ TEST_CASE("simple expire z10 bounds 1023, 0", "[NoDB]") expire_config_t{}); auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom); - CHECK(tiles.size() == 1); + CHECK(tiles.size() == 2); auto itr = tiles.cbegin(); + CHECK(*(itr++) == tile_t(10, 0, 0)); CHECK(*(itr++) == tile_t(10, 1023, 0)); } @@ -205,9 +206,10 @@ TEST_CASE("simple expire z10 bounds 1023, 1023", "[NoDB]") expire_config_t{}); auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom); - CHECK(tiles.size() == 1); + CHECK(tiles.size() == 2); auto itr = tiles.cbegin(); + CHECK(*(itr++) == tile_t(10, 0, 1023)); CHECK(*(itr++) == tile_t(10, 1023, 1023)); } From be86625d9194d76da1ee400872f79275522d71c3 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 24 Oct 2025 16:44:11 +0200 Subject: [PATCH 03/45] Fix: Always disable expiry if maxzoom==0 maxzoom==0 is the default setting and it is documented to means that there is no expire. That's what the enabled() function checks. But disabling this in from_bbox() only disabled the expire for that case, not for linestrings. Better to do this check in the function that's called from outside the expire code: from_geometry(). We also need to fix some tests that were using maxzoom==0 by changing their setting to use maxzoom==1. --- src/expire-tiles.cpp | 8 ++++---- tests/bdd/flex/expire.feature | 8 ++++---- tests/data/test_expire.lua | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/expire-tiles.cpp b/src/expire-tiles.cpp index 8ad2e6c63..b348cbeb8 100644 --- a/src/expire-tiles.cpp +++ b/src/expire-tiles.cpp @@ -147,6 +147,10 @@ void expire_tiles_t::from_geometry(geom::multipolygon_t const &geom, void expire_tiles_t::from_geometry(geom::geometry_t const &geom, expire_config_t const &expire_config) { + if (!enabled()) { + return; + } + geom.visit([&](auto const &g) { from_geometry(g, expire_config); }); } @@ -224,10 +228,6 @@ void expire_tiles_t::from_line_segment(geom::point_t const &a, int expire_tiles_t::from_bbox(geom::box_t const &box, expire_config_t const &expire_config) { - if (!enabled()) { - return 0; - } - double const width = box.width(); double const height = box.height(); if (width > tile_t::HALF_EARTH_CIRCUMFERENCE + 1) { diff --git a/tests/bdd/flex/expire.feature b/tests/bdd/flex/expire.feature index 2230cf7a4..40c6f6f15 100644 --- a/tests/bdd/flex/expire.feature +++ b/tests/bdd/flex/expire.feature @@ -1,4 +1,4 @@ -Feature: Changes on way with expire on zoom 0 +Feature: Changes on way with expire on zoom 1 Background: Given the style file 'test_expire.lua' @@ -71,7 +71,7 @@ Feature: Changes on way with expire on zoom 0 | 11 | Then table osm2pgsql_test_expire contains exactly | zoom | x | y | - | 0 | 0 | 0 | + | 1 | 1 | 0 | Scenario: change in t1 @@ -88,7 +88,7 @@ Feature: Changes on way with expire on zoom 0 | way_id | Then table osm2pgsql_test_expire contains exactly | zoom | x | y | - | 0 | 0 | 0 | + | 1 | 1 | 0 | Scenario: remove from t1 @@ -105,4 +105,4 @@ Feature: Changes on way with expire on zoom 0 | way_id | Then table osm2pgsql_test_expire contains exactly | zoom | x | y | - | 0 | 0 | 0 | + | 1 | 1 | 0 | diff --git a/tests/data/test_expire.lua b/tests/data/test_expire.lua index bd949e743..04b2fe234 100644 --- a/tests/data/test_expire.lua +++ b/tests/data/test_expire.lua @@ -1,7 +1,7 @@ --- No maxzoom sets it to 0 local eo = osm2pgsql.define_expire_output({ table = 'osm2pgsql_test_expire', + maxzoom = 1, }) local the_table = osm2pgsql.define_way_table('osm2pgsql_test_t1', { From 3984965378b8eb7eff5c3483f5dc4a71080967ed Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 27 Oct 2025 15:05:38 +0100 Subject: [PATCH 04/45] osm2pgsql-expire: Expire type=boundary relations as multipolygons You can always use --mode=boundary_only to switch back to boundary only interepretation. So type=boundary is now treated exactly like type=multipolygon. Also adds more information to man page. --- man/osm2pgsql-expire.1 | 75 ++++++++++++++++++++++++++-------------- man/osm2pgsql-expire.md | 22 +++++++++++- src/osm2pgsql-expire.cpp | 2 +- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/man/osm2pgsql-expire.1 b/man/osm2pgsql-expire.1 index c6dbd023c..5425c3fa2 100644 --- a/man/osm2pgsql-expire.1 +++ b/man/osm2pgsql-expire.1 @@ -1,79 +1,102 @@ .TH "OSM2PGSQL" "1" "2.2.0" "" "" .SH NAME -osm2pgsql\-expire \- Visualize expire output +.PP +osm2pgsql-expire - Visualize expire output .SH SYNOPSIS -\f[B]osm2pgsql\-expire\f[R] [\f[I]OPTIONS\f[R]] \f[I]OSM\-FILE\f[R] (1) -\f[B]osm2pgsql\-expire\f[R] \f[I]TILES\-FILE\f[R] (2) +.PP +\f[B]osm2pgsql-expire\f[R] [\f[I]OPTIONS\f[R]] \f[I]OSM-FILE\f[R] (1) +\f[B]osm2pgsql-expire\f[R] \f[I]TILES-FILE\f[R] (2) .SH DESCRIPTION +.PP \f[B]This command is currently experimental.\f[R] .PP The expire command can be used for two things: .IP "1." 3 \f[B]To check what tiles some OSM data is in.\f[R] If an -\f[I]OSM\-FILE\f[R] is specified osm2pgsql\-expire will calculate the +\f[I]OSM-FILE\f[R] is specified osm2pgsql-expire will calculate the tiles covering the objects in that file. Note that the file must not be a change file but a regular OSM data file! Output is, by default, a tile file, but GeoJSON is also possible. +Covering tiles are only calculated for tagged objects. .IP "2." 3 -\f[B]Visualize tile list.\f[R] If a \f[I]TILE\-FILE\f[R] (presumably +\f[B]Visualize tile list.\f[R] If a \f[I]TILE-FILE\f[R] (presumably generated by osm2pgsql) is specified, a GeoJSON file is generated showing all mentioned tiles. In this mode all command line options are ignored. .PP Read the \f[I]Expire\f[R] chapter of the osm2pgsql manual (https://osm2pgsql.org/doc/manual.html#expire) for details on how to -interpret the \f[CR]\-m, \[rs]\-\-mode\f[R] and -\f[CR]\[rs]\-\-full\-area\-limit\f[R] options. +interpret the \f[V]-m, \[rs]--mode\f[R] and +\f[V]\[rs]--full-area-limit\f[R] options. +.SH GEOMETRY GENERATION +.PP +To calculate the tiles covering the specified OSM data in mode (1), a +geometry has to be generated first for each object. +This works as follow: +.IP \[bu] 2 +For nodes a point geometry will be generated. +.IP \[bu] 2 +For closed ways a polygon geometry will be generated, for non-closed +ways a linestring geometry. +If a generated polygon is not valid, the program will fall back to a +linestring. +If neither a valid polygon nor a valid linestring can be generated the +object is ignored. +.IP \[bu] 2 +For relations a geometry is generated based on the \f[V]type\f[R] tag. +For types \f[V]multipolygon\f[R] and \f[V]boundary\f[R], a multipolygon +geometry is created, for types \f[V]route\f[R] and +\f[V]multilinestring\f[R] a multilinestring geometry is created, for all +other types a geometry collection is created. +.PP +Note that for (multi)polygons it depends on the command line options +--mode and --full-area-limit whether the tiles calculation is based on +the area or the boundary of the (multi)polygons. .SH OPTIONS +.PP This program follows the usual GNU command line syntax, with long -options starting with two dashes (\f[CR]\-\-\f[R]). +options starting with two dashes (\f[V]--\f[R]). Mandatory arguments to long options are mandatory for short options too. .SH MAIN OPTIONS .TP -\-b, \-\-buffer=VALUE +-b, --buffer=VALUE Set buffer size around geometry relative to tile size. Example: Set to 0.1 for a buffer that\[cq]s 10% of the tile size. .TP -\-f, \-\-format=FORMAT +-f, --format=FORMAT Output format. Options are `tiles' (default) or `geojson'. The GeoJSON output uses the Web Mercator projection (EPSG:3857) which is supported by many programs although, strictly speaking, it is not allowed by the GeoJSON spec. .TP -\-\-full\-area\-limit=VALUE +--full-area-limit=VALUE Set full area limit. .TP -\-m, \-\-mode=MODE +-m, --mode=MODE Set expire mode. -One of \f[CR]boundary_only\f[R], \f[CR]full_area\f[R] (default), and -\f[CR]hybrid\f[R]. +One of \f[V]boundary_only\f[R], \f[V]full_area\f[R] (default), and +\f[V]hybrid\f[R]. .TP -\-z, \-\-zoom=ZOOM +-z, --zoom=ZOOM Zoom level on which to calculate tiles. .SH HELP/VERSION OPTIONS .TP -\-h, \-\-help +-h, --help Print help. .TP -\-V, \-\-version +-V, --version Print osm2pgsql version. .SH LOGGING OPTIONS .TP -\-\-log\-level=LEVEL +--log-level=LEVEL Set log level (`debug', `info' (default), `warn', or `error'). .SH SEE ALSO .IP \[bu] 2 -\c -.UR https://osm2pgsql.org -osm2pgsql website -.UE \c +osm2pgsql website (https://osm2pgsql.org) .IP \[bu] 2 -\c -.UR https://osm2pgsql.org/doc/manual.html -osm2pgsql manual -.UE \c +osm2pgsql manual (https://osm2pgsql.org/doc/manual.html) .IP \[bu] 2 \f[B]osm2pgsql\f[R](1) .IP \[bu] 2 diff --git a/man/osm2pgsql-expire.md b/man/osm2pgsql-expire.md index 3024b81b0..9c031ee5b 100644 --- a/man/osm2pgsql-expire.md +++ b/man/osm2pgsql-expire.md @@ -16,7 +16,8 @@ The expire command can be used for two things: 1. **To check what tiles some OSM data is in.** If an *OSM-FILE* is specified osm2pgsql-expire will calculate the tiles covering the objects in that file. Note that the file must not be a change file but a regular OSM data file! - Output is, by default, a tile file, but GeoJSON is also possible. + Output is, by default, a tile file, but GeoJSON is also possible. Covering + tiles are only calculated for tagged objects. 2. **Visualize tile list.** If a *TILE-FILE* (presumably generated by osm2pgsql) is specified, a GeoJSON file is generated showing all mentioned tiles. In this mode all command line options are ignored. @@ -25,6 +26,25 @@ Read the *Expire* chapter of the osm2pgsql manual (https://osm2pgsql.org/doc/manual.html#expire) for details on how to interpret the `-m, \--mode` and `\--full-area-limit` options. +# GEOMETRY GENERATION + +To calculate the tiles covering the specified OSM data in mode (1), a geometry +has to be generated first for each object. This works as follow: + +* For nodes a point geometry will be generated. +* For closed ways a polygon geometry will be generated, for non-closed ways + a linestring geometry. If a generated polygon is not valid, the program + will fall back to a linestring. If neither a valid polygon nor a valid + linestring can be generated the object is ignored. +* For relations a geometry is generated based on the `type` tag. For types + `multipolygon` and `boundary`, a multipolygon geometry is created, for types + `route` and `multilinestring` a multilinestring geometry is created, for + all other types a geometry collection is created. + +Note that for (multi)polygons it depends on the command line options \--mode +and \--full-area-limit whether the tiles calculation is based on the area or +the boundary of the (multi)polygons. + # OPTIONS This program follows the usual GNU command line syntax, with long options diff --git a/src/osm2pgsql-expire.cpp b/src/osm2pgsql-expire.cpp index d9616f869..fff07589b 100644 --- a/src/osm2pgsql-expire.cpp +++ b/src/osm2pgsql-expire.cpp @@ -189,7 +189,7 @@ void output_expire_t::relation_add(osmium::Relation const &relation) osmium::memory::Buffer tmp_buffer{1024, osmium::memory::Buffer::auto_grow::yes}; geom::geometry_t geom; - if (type == "multipolygon") { + if (type == "multipolygon" || type == "boundary") { log_debug("Creating multipolygon from relation {}...", relation.id()); geom::create_multipolygon(&geom, relation, buffer, &tmp_buffer); } else if (type == "route" || type == "multilinestring") { From 47938815da5ae734d113d7fb62af64fbb85bfd80 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Thu, 30 Oct 2025 14:28:05 +0100 Subject: [PATCH 05/45] Flex: Allow creating point geometry from way This allows the as_point() function to be called for ways. The function has a single (optional) parameter specifying which member node to create the point from. The index parameter is 1-based as usual in Lua. Negative indexes count from the back, :as_point(), :as_point(nil), :as_point(1) will return the first node, :as_point(2) the second node and so on. :as_point(-1) will return the last node. If the index is 0 or out of range a null geometry will be returned. If the parameter is not a number an error will be generated. --- src/geom-from-osm.cpp | 13 +++- src/geom-from-osm.hpp | 9 +++ src/output-flex.cpp | 41 +++++++++++- tests/bdd/flex/geometry-point.feature | 95 +++++++++++++++++++++++++++ tests/test-geom-linestrings.cpp | 44 +++++++++++++ tests/test-geom-points.cpp | 14 ++++ 6 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 tests/bdd/flex/geometry-point.feature diff --git a/src/geom-from-osm.cpp b/src/geom-from-osm.cpp index c1f0f10d8..e7002d29a 100644 --- a/src/geom-from-osm.cpp +++ b/src/geom-from-osm.cpp @@ -19,11 +19,18 @@ namespace geom { +void create_point(geometry_t *geom, osmium::Location const &location) +{ + if (location.valid()) { + auto &point = geom->set(); + point.set_x(location.lon()); + point.set_y(location.lat()); + } +} + void create_point(geometry_t *geom, osmium::Node const &node) { - auto &point = geom->set(); - point.set_x(node.location().lon()); - point.set_y(node.location().lat()); + create_point(geom, node.location()); } geometry_t create_point(osmium::Node const &node) diff --git a/src/geom-from-osm.hpp b/src/geom-from-osm.hpp index 4db2c8654..0d295cebf 100644 --- a/src/geom-from-osm.hpp +++ b/src/geom-from-osm.hpp @@ -27,6 +27,15 @@ namespace geom { +/** + * Create a point geometry from a location. If the location is not valid, + * the output will not be changed. + * + * \param geom Pointer to an existing geometry which will be used as output. + * \param location The input location. + */ +void create_point(geometry_t *geom, osmium::Location const &location); + /** * Create a point geometry from a node. * diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 5c27e901d..3b554e25e 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -454,11 +454,46 @@ int output_flex_t::app_get_bbox() int output_flex_t::app_as_point() { - check_context_and_state("as_point", "node", - m_calling_context != calling_context::process_node); + check_for_object(lua_state(), "as_point"); + + if (m_calling_context == calling_context::process_node) { + if (lua_gettop(lua_state()) > 1) { + throw fmt_error("No parameter(s) needed for as_point()."); + } + auto *geom = create_lua_geometry_object(lua_state()); + geom::create_point(geom, *m_context_node); + return 1; + } + + if (m_calling_context != calling_context::process_way) { + throw fmt_error( + "The function as_point() can only be called (directly " + "or indirectly) from the process_[untagged]_node/way() functions."); + } + + if (lua_gettop(lua_state()) > 2) { + throw fmt_error("Too many arguments for function as_point()"); + } + + m_way_cache.add_nodes(middle()); + auto const &nodes = m_way_cache.get().nodes(); + + int64_t n = 1; // get first node by default + if (lua_gettop(lua_state()) > 1) { + if (lua_type(lua_state(), 2) != LUA_TNUMBER) { + throw std::runtime_error{ + "Argument #1 to 'as_point()' must be an integer."}; + } + n = lua_tointeger(lua_state(), 2); + if (n < 0) { // negative index values count from the back + n += static_cast(nodes.size()) + 1; + } + } auto *geom = create_lua_geometry_object(lua_state()); - geom::create_point(geom, *m_context_node); + if (n > 0 && static_cast(n) <= nodes.size()) { + geom::create_point(geom, nodes[n - 1].location()); + } // fall through returning null geometry if 0 or too large or small return 1; } diff --git a/tests/bdd/flex/geometry-point.feature b/tests/bdd/flex/geometry-point.feature new file mode 100644 index 000000000..493c246b6 --- /dev/null +++ b/tests/bdd/flex/geometry-point.feature @@ -0,0 +1,95 @@ +Feature: Creating point features from way + + Scenario: + Given the grid + | 1 | 2 | | + | 4 | | 3 | + | | 5 | | + And the OSM data + """ + w20 Thighway=motorway Nn1,n2,n3 + w21 Thighway=motorway Nn4,n5 + """ + And the lua style + """ + local points = osm2pgsql.define_way_table('osm2pgsql_test_points', { + { column = 'n', type = 'int' }, + { column = 'geom', type = 'point', projection = 4326, not_null = false }, + }) + + function osm2pgsql.process_way(object) + if object.tags.highway == 'motorway' then + points:insert({ n = nil, geom = object:as_point() }) + points:insert({ n = 0, geom = object:as_point(0) }) + points:insert({ n = 1, geom = object:as_point(1) }) + points:insert({ n = 2, geom = object:as_point(2) }) + points:insert({ n = 3, geom = object:as_point(3) }) + points:insert({ n = 4, geom = object:as_point(4) }) + points:insert({ n = -1, geom = object:as_point(-1) }) + points:insert({ n = -2, geom = object:as_point(-2) }) + points:insert({ n = -3, geom = object:as_point(-3) }) + end + end + """ + When running osm2pgsql flex + + Then table osm2pgsql_test_points contains exactly + | way_id | n | ST_AsText(geom) | + | 20 | NULL | 1 | + | 20 | 0 | NULL | + | 20 | 1 | 1 | + | 20 | 2 | 2 | + | 20 | 3 | 3 | + | 20 | 4 | NULL | + | 20 | -1 | 3 | + | 20 | -2 | 2 | + | 20 | -3 | 1 | + | 21 | NULL | 4 | + | 21 | 0 | NULL | + | 21 | 1 | 4 | + | 21 | 2 | 5 | + | 21 | 3 | NULL | + | 21 | 4 | NULL | + | 21 | -1 | 5 | + | 21 | -2 | 4 | + | 21 | -3 | NULL | + + Scenario: + Given the grid + | 1 | 2 | + And the OSM data + """ + w20 Thighway=motorway Nn1,n2 + """ + And the lua style + """ + function osm2pgsql.process_way(object) + local geom = object:as_point('foo') + end + """ + Then running osm2pgsql flex fails + + And the error output contains + """ + Argument #1 to 'as_point()' must be an integer. + """ + + Scenario: + Given the grid + | 1 | 2 | + And the OSM data + """ + w20 Thighway=motorway Nn1,n2 + """ + And the lua style + """ + function osm2pgsql.process_way(object) + local geom = object:as_point(1, 'foo') + end + """ + Then running osm2pgsql flex fails + + And the error output contains + """ + Too many arguments for function as_point() + """ diff --git a/tests/test-geom-linestrings.cpp b/tests/test-geom-linestrings.cpp index 33420de6f..b50de68ef 100644 --- a/tests/test-geom-linestrings.cpp +++ b/tests/test-geom-linestrings.cpp @@ -115,6 +115,50 @@ TEST_CASE("create_linestring from invalid OSM data", "[NoDB]") REQUIRE(geom.is_null()); } +TEST_CASE("create_point from OSM way data", "[NoDB]") +{ + test_buffer_t buffer; + buffer.add_way("w20 Nn1x1y1,n2x2y2"); + + auto const &nodes = buffer.buffer().get(0).nodes(); + geom::geometry_t geom; + geom::create_point(&geom, nodes[0].location()); + + REQUIRE(geom.is_point()); + REQUIRE(geometry_type(geom) == "POINT"); + REQUIRE(dimension(geom) == 0); + REQUIRE(num_geometries(geom) == 1); + REQUIRE(geom.get() == geom::point_t{1, 1}); +} + +TEST_CASE("create_point from OSM data without locations", "[NoDB]") +{ + test_buffer_t buffer; + buffer.add_way("w20 Nn1,n2"); + + auto const &nodes = buffer.buffer().get(0).nodes(); + geom::geometry_t geom; + geom::create_point(&geom, nodes[0].location()); + + REQUIRE(geom.is_null()); +} + +TEST_CASE("create_point from way with single node", "[NoDB]") +{ + test_buffer_t buffer; + buffer.add_way("w20 Nn1x1y1"); + + auto const &nodes = buffer.buffer().get(0).nodes(); + geom::geometry_t geom; + geom::create_point(&geom, nodes[0].location()); + + REQUIRE(geom.is_point()); + REQUIRE(geometry_type(geom) == "POINT"); + REQUIRE(dimension(geom) == 0); + REQUIRE(num_geometries(geom) == 1); + REQUIRE(geom.get() == geom::point_t{1, 1}); +} + TEST_CASE("geom::segmentize w/o split", "[NoDB]") { geom::linestring_t const expected{{0, 0}, {1, 2}, {2, 2}}; diff --git a/tests/test-geom-points.cpp b/tests/test-geom-points.cpp index 13b07e8d4..94e5e0418 100644 --- a/tests/test-geom-points.cpp +++ b/tests/test-geom-points.cpp @@ -42,6 +42,20 @@ TEST_CASE("geom::point_t from location", "[NoDB]") REQUIRE(p == geom::point_t{3.141, 2.718}); } +TEST_CASE("geom::point_t from location with create_point", "[NoDB]") +{ + osmium::Location const location{1.1, 2.2}; + + geom::geometry_t geom; + geom::create_point(&geom, location); + REQUIRE(geom.is_point()); + + auto const &p = geom.get(); + REQUIRE(p.x() == Approx(1.1)); + REQUIRE(p.y() == Approx(2.2)); + REQUIRE(p == geom::point_t{1.1, 2.2}); +} + TEST_CASE("create_point from OSM data", "[NoDB]") { test_buffer_t buffer; From 35ef2de952c4bb41fcec2919465ea46a4c9a6008 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 6 Nov 2025 16:53:52 +0100 Subject: [PATCH 06/45] bdd: change handling of untagged nodes Nodes defined in the grid are now added to the import data only when the grid is defined, not when the final OPL is generated. This avoids untagged nodes being added again when running updates. --- tests/bdd/flex/expire.feature | 5 ----- tests/bdd/flex/locator.feature | 3 +-- tests/bdd/flex/node-add.feature | 18 +++++---------- tests/bdd/flex/relation-changes.feature | 27 ++++++++-------------- tests/bdd/flex/way-add.feature | 18 +++++---------- tests/bdd/flex/way-change.feature | 15 +++++-------- tests/bdd/flex/way-del.feature | 24 +++++++------------- tests/bdd/flex/way-relation-add.feature | 30 +++++++++---------------- tests/bdd/flex/way-relation-del.feature | 15 +++++-------- tests/bdd/steps/geometry_factory.py | 30 +++++++++---------------- tests/bdd/steps/steps_execute.py | 2 -- tests/bdd/steps/steps_osm_data.py | 26 ++++++++++++--------- 12 files changed, 76 insertions(+), 137 deletions(-) diff --git a/tests/bdd/flex/expire.feature b/tests/bdd/flex/expire.feature index 2230cf7a4..623cdd794 100644 --- a/tests/bdd/flex/expire.feature +++ b/tests/bdd/flex/expire.feature @@ -26,7 +26,6 @@ Feature: Changes on way with expire on zoom 0 """ w10 v1 dV Ta=b Nn10,n11 """ - And an empty grid When running osm2pgsql flex with parameters | --slim | -a | @@ -43,7 +42,6 @@ Feature: Changes on way with expire on zoom 0 """ n1 v2 dV x1 y2 """ - And an empty grid When running osm2pgsql flex with parameters | --slim | -a | @@ -60,7 +58,6 @@ Feature: Changes on way with expire on zoom 0 """ w10 v1 dV Tt1=yes Nn10,n11 """ - And an empty grid When running osm2pgsql flex with parameters | --slim | -a | @@ -79,7 +76,6 @@ Feature: Changes on way with expire on zoom 0 """ w11 v2 dV Ta=b Nn10,n11 """ - And an empty grid When running osm2pgsql flex with parameters | --slim | -a | @@ -96,7 +92,6 @@ Feature: Changes on way with expire on zoom 0 """ w11 v2 dD """ - And an empty grid When running osm2pgsql flex with parameters | --slim | -a | diff --git a/tests/bdd/flex/locator.feature b/tests/bdd/flex/locator.feature index 7a97b6b8f..1e1bf9aec 100644 --- a/tests/bdd/flex/locator.feature +++ b/tests/bdd/flex/locator.feature @@ -199,8 +199,7 @@ Feature: Locators | way_id | region | ST_AsText(geom) | | 20 | P1 | (10 0,20 10,10 10,10 0) | - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Tamenity=post_box x15.0 y8.0 n11 v1 dV Tamenity=post_box x15.0 y2.0 diff --git a/tests/bdd/flex/node-add.feature b/tests/bdd/flex/node-add.feature index 178ba81b1..f5790dc3c 100644 --- a/tests/bdd/flex/node-add.feature +++ b/tests/bdd/flex/node-add.feature @@ -27,8 +27,7 @@ Feature: Adding nodes to a flex database Scenario: node is not relevant - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Tt=ag x0 y0 r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark @@ -49,8 +48,7 @@ Feature: Adding nodes to a flex database Scenario: add to t1 - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Tt1=yes x0 y0 r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark @@ -72,8 +70,7 @@ Feature: Adding nodes to a flex database Scenario: add to t2 - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Tt2=yes x0 y0 r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark @@ -95,8 +92,7 @@ Feature: Adding nodes to a flex database Scenario: add to t1 and t2 - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Tt1=yes,t2=yes x0 y0 r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark @@ -119,8 +115,7 @@ Feature: Adding nodes to a flex database Scenario: add to tboth (only stage1) - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Ttboth=yes x0 y0 r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark @@ -142,8 +137,7 @@ Feature: Adding nodes to a flex database Scenario: add to tboth (stage1 and stage2) - Given an empty grid - And the OSM data + Given the OSM data """ n10 v1 dV Ttboth=yes x0 y0 r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark diff --git a/tests/bdd/flex/relation-changes.feature b/tests/bdd/flex/relation-changes.feature index a05c469e7..4f16d6c65 100644 --- a/tests/bdd/flex/relation-changes.feature +++ b/tests/bdd/flex/relation-changes.feature @@ -32,8 +32,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 0 rows - Given an empty grid - And the OSM data + Given the OSM data """ r30 v2 dV Ttype=multipolygon Mw20@,w21@ """ @@ -56,8 +55,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 0 rows - Given an empty grid - And the OSM data + Given the OSM data """ w21 v2 dV Nn12,n13,n10 """ @@ -79,8 +77,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 0 rows - Given an empty grid - And the OSM data + Given the OSM data """ n12 v2 dV x10.1 y10.1 """ @@ -103,8 +100,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 0 rows - Given an empty grid - And the OSM data + Given the OSM data """ r30 v2 dV Ttype=multipolygon Mw20@,w21@ """ @@ -127,8 +123,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 1 row - Given an empty grid - And the OSM data + Given the OSM data """ r30 v2 dV Mw20@,w21@ """ @@ -151,8 +146,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 1 row - Given an empty grid - And the OSM data + Given the OSM data """ w21 v2 dV """ @@ -180,8 +174,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 1 row - Given an empty grid - And the OSM data + Given the OSM data """ n12 v2 dV """ @@ -209,8 +202,7 @@ Feature: Handling changes to relations | --slim | Then table osm2pgsql_test_relations has 1 row - Given an empty grid - And the OSM data + Given the OSM data """ r30 v2 dV Ttype=multipolygon """ @@ -240,8 +232,7 @@ Feature: Handling changes to relations | area_id | tags->'natural' | tags->'landuse' | | -30 | wood | NULL | - Given an empty grid - And the OSM data + Given the OSM data """ r30 v2 dV Ttype=multipolygon,landuse=forest Mw20@,w21@ """ diff --git a/tests/bdd/flex/way-add.feature b/tests/bdd/flex/way-add.feature index bad8041ef..4b7ca86a1 100644 --- a/tests/bdd/flex/way-add.feature +++ b/tests/bdd/flex/way-add.feature @@ -31,8 +31,7 @@ Feature: Adding ways to a flex database Scenario: way is not relevant - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Tt=ag Nn10,n11 r30 v2 dV Tt=ag Mw10@,w11@,w12@mark,w13@,w14@mark @@ -53,8 +52,7 @@ Feature: Adding ways to a flex database Scenario: add to t1 - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Tt1=yes Nn10,n11 r30 v2 dV Tt=ag Mw10@,w11@,w12@mark,w13@,w14@mark @@ -76,8 +74,7 @@ Feature: Adding ways to a flex database Scenario: add to t2 - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Tt2=yes Nn10,n11 r30 v2 dV Tt=ag Mw10@mark,w11@,w12@mark,w13@,w14@mark @@ -99,8 +96,7 @@ Feature: Adding ways to a flex database Scenario: add to t1 and t2 - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Tt1=yes,t2=yes Nn10,n11 r30 v2 dV Tt=ag Mw10@mark,w11@,w12@mark,w13@,w14@mark @@ -123,8 +119,7 @@ Feature: Adding ways to a flex database Scenario: add to tboth (only stage1) - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Ttboth=yes Nn10,n11 r30 v2 dV Tt=ag Mw10@,w11@,w12@mark,w13@,w14@mark @@ -146,8 +141,7 @@ Feature: Adding ways to a flex database Scenario: add to tboth (stage1 and stage2) - Given an empty grid - And the OSM data + Given the OSM data """ w10 v1 dV Ttboth=yes Nn10,n11 r30 v2 dV Tt=ag Mw10@mark,w11@,w12@mark,w13@,w14@mark diff --git a/tests/bdd/flex/way-change.feature b/tests/bdd/flex/way-change.feature index 941c8d319..4346fec50 100644 --- a/tests/bdd/flex/way-change.feature +++ b/tests/bdd/flex/way-change.feature @@ -31,8 +31,7 @@ Feature: Changing ways in a flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -84,8 +83,7 @@ Feature: Changing ways in a flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -138,8 +136,7 @@ Feature: Changing ways in a flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -196,8 +193,7 @@ Feature: Changing ways in a flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -243,8 +239,7 @@ Feature: Changing ways in a flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dV Tt1=yes,t2=yes,tboth=yes Nn10,n11 """ diff --git a/tests/bdd/flex/way-del.feature b/tests/bdd/flex/way-del.feature index 0e650e885..bc0d25d17 100644 --- a/tests/bdd/flex/way-del.feature +++ b/tests/bdd/flex/way-del.feature @@ -31,8 +31,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -74,8 +73,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -117,8 +115,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -159,8 +156,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -202,8 +198,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -245,8 +240,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -289,8 +283,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ @@ -332,8 +325,7 @@ Feature: Deleting ways in a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ w10 v2 dD """ diff --git a/tests/bdd/flex/way-relation-add.feature b/tests/bdd/flex/way-relation-add.feature index 984f7862b..3ec232b37 100644 --- a/tests/bdd/flex/way-relation-add.feature +++ b/tests/bdd/flex/way-relation-add.feature @@ -34,8 +34,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -86,8 +85,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -133,8 +131,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -180,8 +177,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -231,8 +227,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -277,8 +272,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@,w11@,w12@,w13@,w14@,w15@ """ @@ -328,8 +322,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@mark,w11@,w12@,w13@,w14@,w15@ """ @@ -375,8 +368,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@,w11@,w12@,w13@,w14@,w15@ """ @@ -422,8 +414,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@,w11@,w12@,w13@,w14@,w15@ """ @@ -468,8 +459,7 @@ Feature: Adding relations to a 2-stage flex database | 13 | NULL | | 14 | {30} | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dV Tt=ag Mw10@,w11@,w12@,w13@,w14@,w15@ """ diff --git a/tests/bdd/flex/way-relation-del.feature b/tests/bdd/flex/way-relation-del.feature index 1efc34ef0..eb15b807d 100644 --- a/tests/bdd/flex/way-relation-del.feature +++ b/tests/bdd/flex/way-relation-del.feature @@ -33,8 +33,7 @@ Feature: Deleting relations in a stage-2 flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dD """ @@ -78,8 +77,7 @@ Feature: Deleting relations in a stage-2 flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ r32 v2 dD """ @@ -125,8 +123,7 @@ Feature: Deleting relations in a stage-2 flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -176,8 +173,7 @@ Feature: Deleting relations in a stage-2 flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ @@ -228,8 +224,7 @@ Feature: Deleting relations in a stage-2 flex database | 13 | | 14 | - Given an empty grid - And the OSM data + Given the OSM data """ """ diff --git a/tests/bdd/steps/geometry_factory.py b/tests/bdd/steps/geometry_factory.py index 41e3f18ee..b56361da6 100644 --- a/tests/bdd/steps/geometry_factory.py +++ b/tests/bdd/steps/geometry_factory.py @@ -64,10 +64,6 @@ def mk_wkt_points(self, geom): return ','.join([self.mk_wkt_point(x) for x in geom.split(',')]) - def remove_grid(self): - self.grid = {} - - def set_grid(self, lines, grid_step, origin_x, origin_y): """ Replace the grid with one from the given lines. """ @@ -92,23 +88,17 @@ def grid_node(self, nodeid): """ return self.grid.get(nodeid) + def as_opl_lines(self): + return [f"n{nid} x{c[0]:.{self.grid_precision}f} y{c[1]:.{self.grid_precision}f}" + for nid, c in self.grid.items()] - def complete_node_list(self, nodes): - todos = set(self.grid.keys()) - - for i in range(len(nodes)): - line = nodes[i] - nid = int(line[1:].split(' ')[0]) - - if ' x' not in line: - assert ' y' not in line + def complete_opl(self, opl): + if opl[0] != 'n' or ' x' in opl: + return opl - coords = self.grid_node(nid) - assert coords is not None, f"Coordinates missing for node {nid}" - nodes[i] = f"{line} x{coords[0]:.{self.grid_precision}f} y{coords[1]:.{self.grid_precision}f}" + nid = int(opl.split(' ', 1)[0][1:]) - todos.discard(nid) + coords = self.grid_node(nid) + assert coords is not None, f"Coordinates missing for node {nid}" - for nid in todos: - coords = self.grid_node(nid) - nodes.append(f"n{nid} x{coords[0]:.{self.grid_precision}f} y{coords[1]:.{self.grid_precision}f}") + return f"{opl} x{coords[0]:.{self.grid_precision}f} y{coords[1]:.{self.grid_precision}f}" diff --git a/tests/bdd/steps/steps_execute.py b/tests/bdd/steps/steps_execute.py index b2d4bd6d6..d8dea1a82 100644 --- a/tests/bdd/steps/steps_execute.py +++ b/tests/bdd/steps/steps_execute.py @@ -21,8 +21,6 @@ def get_import_file(context): if context.import_file is not None: return str(context.import_file), None - context.geometry_factory.complete_node_list(context.import_data['n']) - # sort by OSM id for obj in context.import_data.values(): obj.sort(key=lambda l: int(l.split(' ')[0][1:])) diff --git a/tests/bdd/steps/steps_osm_data.py b/tests/bdd/steps/steps_osm_data.py index 37f44cf69..9dc27e771 100644 --- a/tests/bdd/steps/steps_osm_data.py +++ b/tests/bdd/steps/steps_osm_data.py @@ -8,6 +8,20 @@ Steps for creating the OSM import file. """ +def add_opl_lines(context, lines): + for line in lines: + if (oplobj := line.strip()): + assert oplobj[0] in ('n', 'w', 'r') + oplobj = context.geometry_factory.complete_opl(oplobj) + data = context.import_data[oplobj[0]] + objid = oplobj.split(' ', 1)[0] + ' ' + for i, existing in enumerate(data): + if existing.startswith(objid): + data[i] = oplobj + break + else: + data.append(oplobj) + @given("the input file '(?P.+)'") def osm_set_import_file(context, osm_file): @@ -16,11 +30,6 @@ def osm_set_import_file(context, osm_file): context.import_file = context.test_data_dir / osm_file -@given("an empty grid") -def osm_define_node_grid(context): - context.geometry_factory.remove_grid() - - @given("the (?P[0-9.]+ )?grid(?: with origin (?P[0-9.-]+) (?P[0-9.-]+))?") def osm_define_node_grid(context, step, origin_x, origin_y): step = float(step.strip()) if step else 0.1 @@ -32,6 +41,7 @@ def osm_define_node_grid(context, step, origin_x, origin_y): context.geometry_factory.set_grid([context.table.headings] + [list(h) for h in context.table], step, x, y) + add_opl_lines(context, context.geometry_factory.as_opl_lines()) @given("the (?Ppython-formatted )?OSM data") @@ -41,8 +51,4 @@ def osm_define_data(context, formatted): if formatted: data = eval('f"""' + data + '"""') - for line in data.split('\n'): - line = line.strip() - if line: - assert line[0] in ('n', 'w', 'r') - context.import_data[line[0]].append(line) + add_opl_lines(context, data.split('\n')) From 64b4fc37a7cf2c7209fa0239a675676133480912 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 7 Nov 2025 14:12:32 +0100 Subject: [PATCH 07/45] bdd: replace sum step with tale check with aggregate --- tests/bdd/regression/import.feature | 89 ++++++++++++++++------------- tests/bdd/steps/steps_db.py | 17 ------ 2 files changed, 48 insertions(+), 58 deletions(-) diff --git a/tests/bdd/regression/import.feature b/tests/bdd/regression/import.feature index eb98fafc0..c848dc1f4 100644 --- a/tests/bdd/regression/import.feature +++ b/tests/bdd/regression/import.feature @@ -7,14 +7,15 @@ Feature: Imports of the test database When running osm2pgsql pgsql Then table planet_osm_point has 1342 rows - And table planet_osm_line has 3231 rows - And table planet_osm_roads has 375 rows - And table planet_osm_polygon has 4130 rows - - Then the sum of 'cast(ST_Area(way) as numeric)' in table planet_osm_polygon is 1247245186 - And the sum of 'cast(way_area as numeric)' in table planet_osm_polygon is 1247245413 - And the sum of 'ST_Length(way)' in table planet_osm_line is 4211350 - And the sum of 'ST_Length(way)' in table planet_osm_roads is 2032023 + And table planet_osm_polygon contains + | count(*) | round(sum(ST_Area(way))) | round(sum(way_area)) | + | 4130 | 1247245186 | 1247245413 | + And table planet_osm_line contains + | count(*) | round(sum(ST_Length(way))) | + | 3231 | 4211350 | + And table planet_osm_roads contains + | count(*) | round(sum(ST_Length(way))) | + | 375 | 2032023 | And there are no tables planet_osm_nodes, planet_osm_ways, planet_osm_rels @@ -37,14 +38,15 @@ Feature: Imports of the test database | | Then table planet_osm_point has 1342 rows - And table planet_osm_line has 3231 rows - And table planet_osm_roads has 375 rows - And table planet_osm_polygon has 4130 rows - - Then the sum of 'cast(ST_Area(way) as numeric)' in table planet_osm_polygon is 1247245186 - And the sum of 'cast(way_area as numeric)' in table planet_osm_polygon is 1247245413 - And the sum of 'ST_Length(way)' in table planet_osm_line is 4211350 - And the sum of 'ST_Length(way)' in table planet_osm_roads is 2032023 + And table planet_osm_polygon contains + | count(*) | round(sum(ST_Area(way))) | round(sum(way_area)) | + | 4130 | 1247245186 | 1247245413 | + And table planet_osm_line contains + | count(*) | round(sum(ST_Length(way))) | + | 3231 | 4211350 | + And table planet_osm_roads contains + | count(*) | round(sum(ST_Length(way))) | + | 375 | 2032023 | And there are planet_osm_nodes, planet_osm_ways, planet_osm_rels @@ -61,14 +63,15 @@ Feature: Imports of the test database | | Then table planet_osm_point has 1342 rows - And table planet_osm_line has 3231 rows - And table planet_osm_roads has 375 rows - And table planet_osm_polygon has 4136 rows - - Then the sum of 'cast(ST_Area(way) as numeric)' in table planet_osm_polygon is 1272140688 - And the sum of 'cast(way_area as numeric)' in table planet_osm_polygon is 1272140891 - And the sum of 'ST_Length(way)' in table planet_osm_line is 4211350 - And the sum of 'ST_Length(way)' in table planet_osm_roads is 2032023 + And table planet_osm_polygon contains + | count(*) | round(sum(ST_Area(way))) | round(sum(way_area)) | + | 4136 | 1272140688 | 1272140891 | + And table planet_osm_line contains + | count(*) | round(sum(ST_Length(way))) | + | 3231 | 4211350 | + And table planet_osm_roads contains + | count(*) | round(sum(ST_Length(way))) | + | 375 | 2032023 | And there are planet_osm_nodes, planet_osm_ways, planet_osm_rels @@ -102,15 +105,18 @@ Feature: Imports of the test database When running osm2pgsql pgsql with parameters | --slim | -j | - Then table planet_osm_point has 1360 rows - And table planet_osm_line has 3254 rows - And table planet_osm_roads has 375 rows - And table planet_osm_polygon has 4131 rows - - Then the sum of 'array_length(akeys(tags),1)' in table planet_osm_point is 4228 - And the sum of 'array_length(akeys(tags),1)' in table planet_osm_roads is 2317 - And the sum of 'array_length(akeys(tags),1)' in table planet_osm_line is 10387 - And the sum of 'array_length(akeys(tags),1)' in table planet_osm_polygon is 9538 + Then table planet_osm_point contains + | count(*) | sum(array_length(akeys(tags), 1)) | + | 1360 | 4228 | + Then table planet_osm_line contains + | count(*) | sum(array_length(akeys(tags), 1)) | + | 3254 | 10387 | + Then table planet_osm_roads contains + | count(*) | sum(array_length(akeys(tags), 1)) | + | 375 | 2317 | + Then table planet_osm_polygon contains + | count(*) | sum(array_length(akeys(tags), 1)) | + | 4131 | 9538 | Scenario Outline: Import slim with various tweaks @@ -121,14 +127,15 @@ Feature: Imports of the test database | | Then table planet_osm_point has 1342 rows - And table planet_osm_line has 3231 rows - And table planet_osm_roads has 375 rows - And table planet_osm_polygon has 4130 rows - - Then the sum of 'cast(ST_Area(way) as numeric)' in table planet_osm_polygon is 1247245186 - And the sum of 'cast(way_area as numeric)' in table planet_osm_polygon is 1247245413 - And the sum of 'ST_Length(way)' in table planet_osm_line is 4211350 - And the sum of 'ST_Length(way)' in table planet_osm_roads is 2032023 + And table planet_osm_polygon contains + | count(*) | round(sum(ST_Area(way))) | round(sum(way_area)) | + | 4130 | 1247245186 | 1247245413 | + And table planet_osm_line contains + | count(*) | round(sum(ST_Length(way))) | + | 3231 | 4211350 | + And table planet_osm_roads contains + | count(*) | round(sum(ST_Length(way))) | + | 375 | 2032023 | And there are tables planet_osm_nodes, planet_osm_ways, planet_osm_rels diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index 26159c2be..c2ba93298 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -44,23 +44,6 @@ def db_table_row_count(context, table, row_num, has_where): f"Table {table}: expected {row_num} rows, got {actual}" -@then(r"the sum of '(?P.+)' in table (?P.+) is (?P\d+)(?P with condition)?") -def db_table_sum_up(context, table, formula, result, has_where): - assert table_exists(context.db, table) - - query = sql.SQL("SELECT round(sum({})) FROM {}")\ - .format(sql.SQL(formula), sql.Identifier(*table.split('.', 2))) - - if has_where: - query = sql.SQL("{} WHERE {}").format(query, sql.SQL(context.text)) - - - actual = scalar(context.db, query) - - assert actual == int(result),\ - f"Table {table}: expected sum {result}, got {actual}" - - @then("there (?:is|are) (?Pno )?tables? (?P.+)") def db_table_existance(context, exists, tables): for table in tables.split(','): From 391b2f5d05eed7de4bef5576a885da5d20438686 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 9 Nov 2025 13:08:13 +0100 Subject: [PATCH 08/45] add joto as maintainer --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d0e74fc8..c1c7dd84a 100644 --- a/README.md +++ b/README.md @@ -223,8 +223,9 @@ GNU General Public License for more details. ## Maintainers -The current maintainers of osm2pgsql are [Sarah Hoffmann](https://github.com/lonvia/) -and [Paul Norman](https://github.com/pnorman/). +The current maintainers of osm2pgsql are [Sarah Hoffmann](https://github.com/lonvia/), +[Paul Norman](https://github.com/pnorman/) and +[Jochen Topf](https://github.com/joto). ## Contributing From 2aa48c777c08ecc2a5871df45a7efda2cbd957ac Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 28 Nov 2025 17:24:36 +0100 Subject: [PATCH 09/45] BDD: drop "row with condition" in favour of "doesn't contain" --- tests/bdd/flex/lua-index-definitions.feature | 55 +++++++++---------- tests/bdd/flex/way-change.feature | 46 +++++++--------- .../regression/forward_dependencies.feature | 25 ++++----- tests/bdd/regression/import.feature | 41 ++++---------- tests/bdd/regression/multipolygon.feature | 31 +++++------ tests/bdd/steps/steps_db.py | 24 ++++++-- 6 files changed, 104 insertions(+), 118 deletions(-) diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature index da77db887..eeb92202a 100644 --- a/tests/bdd/flex/lua-index-definitions.feature +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -473,9 +473,10 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%WHERE (name = lower(name))%' - | schemaname | tablename | - | public | mytable | + Then table pg_catalog.pg_indexes contains + | schemaname | tablename | indexdef@fullmatch | + | public | mytable | .*USING btree \(name\).*WHERE \(name = lower\(name\)\).* | + Scenario: Don't create id index if the configuration doesn't mention it Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -492,10 +493,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes has 0 rows with condition - """ - schemaname = 'public' AND tablename = 'mytable' AND indexname LIKE '%node_id%' - """ + Then table pg_catalog.pg_indexes doesn't contain + | schemaname | tablename | indexname@fullmatch | + | public | mytable | .*node_id.* | Scenario: Don't create id index if the configuration doesn't says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -512,10 +512,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes has 0 rows with condition - """ - schemaname = 'public' AND tablename = 'mytable' AND indexname LIKE '%node_id%' - """ + Then table pg_catalog.pg_indexes doesn't contain + | schemaname | tablename | indexname@fullmatch | + | public | mytable | .*node_id.* | Scenario: Always create id index if the configuration says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -532,10 +531,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes has 1 rows with condition - """ - schemaname = 'public' AND tablename = 'mytable' AND indexname LIKE '%node_id%' - """ + Then table pg_catalog.pg_indexes contains + | schemaname | tablename | indexname@fullmatch | + | public | mytable | .*node_id.* | Scenario: Create a unique id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -553,13 +551,12 @@ Feature: Index definitions in Lua file """ When running osm2pgsql flex Then table foo has 1562 rows - And SELECT indexdef FROM pg_indexes WHERE tablename = 'foo' - | indexdef@fullmatch | - | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And table pg_catalog.pg_index has 0 rows with condition - """ - indrelid = 'foo'::regclass and indisprimary - """ + Then table pg_catalog.pg_indexes contains + | tablename | indexdef@fullmatch | + | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | + And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary + | count | + | 0 | Scenario: Create a primary key id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -577,11 +574,9 @@ Feature: Index definitions in Lua file """ When running osm2pgsql flex Then table foo has 1562 rows - And SELECT indexdef FROM pg_indexes WHERE tablename = 'foo' - | indexdef@fullmatch | - | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And table pg_catalog.pg_index has 1 row with condition - """ - indrelid = 'foo'::regclass and indisprimary - """ - + Then table pg_catalog.pg_indexes contains + | tablename | indexdef@fullmatch | + | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | + And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary + | count | + | 1 | diff --git a/tests/bdd/flex/way-change.feature b/tests/bdd/flex/way-change.feature index 4346fec50..28c73d448 100644 --- a/tests/bdd/flex/way-change.feature +++ b/tests/bdd/flex/way-change.feature @@ -40,10 +40,9 @@ Feature: Changing ways in a flex database Then table osm2pgsql_test_t1 contains | way_id | | 11 | - And table osm2pgsql_test_t1 has rows with condition - """ - way_id = 10 - """ + And table osm2pgsql_test_t1 + | way_id | + | 10 | Then table osm2pgsql_test_t2 contains exactly | way_id | | 10 | @@ -54,9 +53,9 @@ Feature: Changing ways in a flex database | 14 | Examples: - | input | num_w10 | - | w10 v2 dV Tt2=yes Nn10,n11 | 0 | - | w10 v2 dV Tt1=yes,t2=yes Nn10,n11 | 1 | + | input | exist_w10 | + | w10 v2 dV Tt2=yes Nn10,n11 | doesn't contain | + | w10 v2 dV Tt1=yes,t2=yes Nn10,n11 | contains | Scenario Outline: change way from t2 @@ -96,19 +95,18 @@ Feature: Changing ways in a flex database Then table osm2pgsql_test_t2 contains | way_id | | 12 | - And table osm2pgsql_test_t2 has rows with condition - """ - way_id = 10 - """ + And table osm2pgsql_test_t2 + | way_id | + | 10 | Then table osm2pgsql_test_tboth contains exactly | way_id | | 13 | | 14 | Examples: - | input | num_w10 | - | w10 v2 dV Tt1=yes Nn10,n11 | 0 | - | w10 v2 dV Tt1=yes,t2=yes Nn10,n11 | 1 | + | input | exist_w10 | + | w10 v2 dV Tt1=yes Nn10,n11 | doesn't contain | + | w10 v2 dV Tt1=yes,t2=yes Nn10,n11 | contains | Scenario Outline: change way from t1 and t2 @@ -145,26 +143,24 @@ Feature: Changing ways in a flex database Then table osm2pgsql_test_t1 contains | way_id | | 11 | - And table osm2pgsql_test_t1 has rows with condition - """ - way_id = 10 - """ + And table osm2pgsql_test_t1 + | way_id | + | 10 | Then table osm2pgsql_test_t2 contains | way_id | | 12 | - And table osm2pgsql_test_t2 has rows with condition - """ - way_id = 10 - """ + And table osm2pgsql_test_t2 + | way_id | + | 10 | Then table osm2pgsql_test_tboth contains exactly | way_id | | 13 | | 14 | Examples: - | input | num_t1 | num_t2 | - | w10 v2 dV Tt1=yes Nn10,n11 | 1 | 0 | - | w10 v2 dV Tt2=yes Nn10,n11 | 0 | 1 | + | input | exist_t1 | exist_t2 | + | w10 v2 dV Tt1=yes Nn10,n11 | contains | doesn't contain | + | w10 v2 dV Tt2=yes Nn10,n11 | doesn't contain | contains | Scenario Outline: change valid geom to invalid geom diff --git a/tests/bdd/regression/forward_dependencies.feature b/tests/bdd/regression/forward_dependencies.feature index 8b48bbee2..cf96032e7 100644 --- a/tests/bdd/regression/forward_dependencies.feature +++ b/tests/bdd/regression/forward_dependencies.feature @@ -29,16 +29,15 @@ Feature: Test forward propagation of changes When running osm2pgsql pgsql with parameters | --slim | -a | --latlong | - Then table planet_osm_point has 1 row - Then table planet_osm_line has 1 row - Then table planet_osm_line has 0 rows with condition - """ - abs(ST_X(ST_StartPoint(way)) - 3.0) < 0.0001 - """ - Then table planet_osm_line has 1 row with condition - """ - abs(ST_X(ST_StartPoint(way)) - 3.1) < 0.0001 - """ - Then table planet_osm_roads has 1 row - Then table planet_osm_polygon has 1 row - + Then table planet_osm_point contains exactly + | osm_id | + | 12 | + Then table planet_osm_line contains exactly + | osm_id | round(ST_X(ST_StartPoint(way))::numeric, 1) | + | 21 | 3.1 | + Then table planet_osm_roads contains exactly + | osm_id | + | 21 | + Then table planet_osm_polygon contains exactly + | osm_id | + | 20 | diff --git a/tests/bdd/regression/import.feature b/tests/bdd/regression/import.feature index c848dc1f4..4cd4760d6 100644 --- a/tests/bdd/regression/import.feature +++ b/tests/bdd/regression/import.feature @@ -162,32 +162,15 @@ Feature: Imports of the test database | -j | | -x | - Then table planet_osm_point has 1360 rows with condition - """ - tags ? 'osm_user' AND - tags ? 'osm_version' AND - tags ? 'osm_uid' AND - tags ? 'osm_changeset' - """ - And table planet_osm_line has 3254 rows with condition - """ - tags ? 'osm_user' AND - tags ? 'osm_version' AND - tags ? 'osm_uid' AND - tags ? 'osm_changeset' - """ - And table planet_osm_roads has 375 rows with condition - """ - tags ? 'osm_user' AND - tags ? 'osm_version' AND - tags ? 'osm_uid' AND - tags ? 'osm_changeset' - """ - And table planet_osm_polygon has 4131 rows with condition - """ - tags ? 'osm_user' AND - tags ? 'osm_version' AND - tags ? 'osm_uid' AND - tags ? 'osm_changeset' - """ - + Then table planet_osm_point contains + | count(*) | every(tags ?& ARRAY['osm_user', 'osm_version', 'osm_uid', 'osm_changeset']) | + | 1360 | True | + Then table planet_osm_line contains + | count(*) | every(tags ?& ARRAY['osm_user', 'osm_version', 'osm_uid', 'osm_changeset']) | + | 3254 | True | + Then table planet_osm_roads contains + | count(*) | every(tags ?& ARRAY['osm_user', 'osm_version', 'osm_uid', 'osm_changeset']) | + | 375 | True | + Then table planet_osm_polygon contains + | count(*) | every(tags ?& ARRAY['osm_user', 'osm_version', 'osm_uid', 'osm_changeset']) | + | 4131 | True | diff --git a/tests/bdd/regression/multipolygon.feature b/tests/bdd/regression/multipolygon.feature index 1e3701219..ce4b48ef5 100644 --- a/tests/bdd/regression/multipolygon.feature +++ b/tests/bdd/regression/multipolygon.feature @@ -54,15 +54,14 @@ Feature: Import and update of multipolygon areas | 138 | 1 | | 140 | 1 | - Then table planet_osm_polygon has 0 rows with condition - """ - osm_id IN (109, 104) - """ + Then table planet_osm_polygon doesn't contain + | osm_id | + | 109 | + | 104 | - Then table planet_osm_polygon has 0 rows with condition - """ - osm_id = -33 and "natural" = 'water' - """ + Then table planet_osm_polygon doesn't contain + | osm_id | "natural" | + | -33 | water | Then SELECT osm_id, CASE WHEN '' = '-G' THEN min(ST_NumGeometries(way)) ELSE count(*) END FROM planet_osm_polygon GROUP BY osm_id | osm_id | count | @@ -118,15 +117,15 @@ Feature: Import and update of multipolygon areas | 138 | 1 | | 140 | 1 | - Then table planet_osm_polygon has 0 rows with condition - """ - osm_id IN (-25, 109, 104) - """ + Then table planet_osm_polygon doesn't contain + | osm_id | + | -25 | + | 109 | + | 104 | - Then table planet_osm_polygon has 0 rows with condition - """ - osm_id = -33 and "natural" = 'water' - """ + Then table planet_osm_polygon doesn't contain + | osm_id | "natural" | + | -33 | water | Examples: | param1 | lua | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index c2ba93298..f83d085af 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -29,15 +29,12 @@ def delete_table(context, table): cur.execute("DROP TABLE " + table) -@then(r"table (?P
.+) has (?P\d+) rows?(?P with condition)?") -def db_table_row_count(context, table, row_num, has_where): +@then(r"table (?P
.+) has (?P\d+) rows?") +def db_table_row_count(context, table, row_num): assert table_exists(context.db, table) query = sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(*table.split('.', 2))) - if has_where: - query = sql.SQL("{} WHERE {}").format(query, sql.SQL(context.text)) - actual = scalar(context.db, query) assert actual == int(row_num),\ @@ -78,6 +75,23 @@ def db_check_table_content(context, table, exact): assert not exact or not actuals,\ f"Unexpected lines in row:\n{actuals}" + +@then("table (?P
.+) doesn't contain") +def db_check_table_absence(context, table): + assert table_exists(context.db, table) + + rows = sql.SQL(', '.join(h.rsplit('@')[0] for h in context.table.headings)) + + with context.db.cursor() as cur: + cur.execute(sql.SQL("SELECT {} FROM {}") + .format(rows, sql.Identifier(*table.split('.', 2)))) + + actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur) + + for row in context.table.rows: + assert not row in actuals, f"Row unexpectedly found: {row}. Full content:\n{actuals}" + + @then("(?PSELECT .*)") def db_check_sql_statement(context, query): with context.db.cursor() as cur: From 3913c6d6753ea5fd1c4fcc18602ddeffcd44555c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:14:42 +0100 Subject: [PATCH 10/45] bdd: use a SQL statement template for index checks Also introduces substring matching. --- tests/bdd/environment.py | 1 + tests/bdd/flex/lua-index-definitions.feature | 121 ++++++++++--------- tests/bdd/steps/steps_db.py | 35 +++++- 3 files changed, 92 insertions(+), 65 deletions(-) diff --git a/tests/bdd/environment.py b/tests/bdd/environment.py index 350b5e1b6..16919ec85 100644 --- a/tests/bdd/environment.py +++ b/tests/bdd/environment.py @@ -122,6 +122,7 @@ def before_scenario(context, scenario): context.geometry_factory = GeometryFactory() context.osm2pgsql_replication.ReplicationServer = ReplicationServerMock() context.urlrequest_responses = {} + context.sql_statements = {} def _mock_urlopen(request): if not request.full_url in context.urlrequest_responses: diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature index eeb92202a..051253fa5 100644 --- a/tests/bdd/flex/lua-index-definitions.feature +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -1,5 +1,16 @@ Feature: Index definitions in Lua file + Background: + Given the SQL statement mytable_indexes + """ + SELECT indexdef, indisprimary as is_primary + FROM pg_catalog.pg_index, pg_catalog.pg_indexes + WHERE schemaname = 'public' + AND tablename = 'mytable' + AND indrelid = tablename::regclass + AND indexrelid = indexname::regclass + """ + Scenario: Indexes field in table definition must be an array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -36,9 +47,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING gist (geom) | Scenario: Empty indexes field in table definition gets you no index Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -56,8 +67,8 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' - | schemaname | tablename | + Then statement mytable_indexes returns exactly + | indexdef | Scenario: Explicitly setting an index column works Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -77,9 +88,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) | Scenario: Explicitly setting multiple indexes Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -101,15 +112,11 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | - And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' - | schemaname | tablename | - | public | mytable | - And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name, tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING btree (name) | + | USING gist (geom) | + | USING btree (name, tags) | Scenario: Method can not be missing Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -244,9 +251,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (lower(name))%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (lower(name)) | @needs-pg-index-includes Scenario: Include field must be a string or array @@ -315,9 +322,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) INCLUDE (tags) | @needs-pg-index-includes Scenario: Include field works with array @@ -338,9 +345,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) INCLUDE (tags) | Scenario: Tablespace needs a string Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -383,9 +390,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) | Scenario: Unique needs a boolean Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -428,9 +435,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%UNIQUE%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@fullmatch | + | .*UNIQUE.*USING btree \(name\).* | Scenario: Where condition needs a string Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -473,9 +480,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes contains - | schemaname | tablename | indexdef@fullmatch | - | public | mytable | .*USING btree \(name\).*WHERE \(name = lower\(name\)\).* | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) WHERE (name = lower(name)) | Scenario: Don't create id index if the configuration doesn't mention it @@ -493,9 +500,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes doesn't contain - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING gist (geom) | Scenario: Don't create id index if the configuration doesn't says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -512,9 +519,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes doesn't contain - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING gist (geom) | Scenario: Always create id index if the configuration says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -531,16 +538,16 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes contains - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (node_id) | Scenario: Create a unique id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style """ local t = osm2pgsql.define_table({ - name = 'foo', + name = 'mytable', ids = { type = 'node', id_column = 'node_id', create_index = 'unique' }, columns = {} }) @@ -550,20 +557,17 @@ Feature: Index definitions in Lua file end """ When running osm2pgsql flex - Then table foo has 1562 rows - Then table pg_catalog.pg_indexes contains - | tablename | indexdef@fullmatch | - | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary - | count | - | 0 | + Then table mytable has 1562 rows + Then statement mytable_indexes returns + | indexdef@fullmatch | is_primary | + | CREATE UNIQUE INDEX .* USING .*\(node_id\).* | False | Scenario: Create a primary key id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style """ local t = osm2pgsql.define_table({ - name = 'foo', + name = 'mytable', ids = { type = 'node', id_column = 'node_id', create_index = 'primary_key' }, columns = {} }) @@ -573,10 +577,7 @@ Feature: Index definitions in Lua file end """ When running osm2pgsql flex - Then table foo has 1562 rows - Then table pg_catalog.pg_indexes contains - | tablename | indexdef@fullmatch | - | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary - | count | - | 1 | + Then table mytable has 1562 rows + Then statement mytable_indexes returns + | indexdef@fullmatch | is_primary | + | CREATE UNIQUE INDEX .* USING .*\(node_id\) | True | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index f83d085af..c91d1e059 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -92,19 +92,30 @@ def db_check_table_absence(context, table): assert not row in actuals, f"Row unexpectedly found: {row}. Full content:\n{actuals}" -@then("(?PSELECT .*)") -def db_check_sql_statement(context, query): +@given("the SQL statement (?P.+)") +def db_define_sql_statement(context, sql): + context.sql_statements[sql] = context.text + +@then("statement (?P.+) returns(?P exactly)?") +def db_check_sql_statement(context, stmt, exact): with context.db.cursor() as cur: - cur.execute(query) + assert stmt in context.sql_statements + cur.execute(context.sql_statements[stmt]) actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur) linenr = 1 for row in context.table.rows: - assert any(r == row for r in actuals),\ - f"{linenr}. entry not found in table. Full content:\n{actuals}" + try: + actuals.remove(row) + except ValueError: + assert False,\ + f"{linenr}. entry not found in result. Full response:\n{actuals}" linenr += 1 + assert not exact or not actuals,\ + f"Unexpected lines in result:\n{actuals}" + ### Helper functions and classes @@ -152,6 +163,8 @@ def __init__(self, row, headings, factory): self.data.append(DBValueGeometry(value, props, factory)) elif props == 'fullmatch': self.data.append(DBValueRegex(value)) + elif props == 'substr': + self.data.append(DBValueSubString(value)) else: self.data.append(str(value)) @@ -310,3 +323,15 @@ def __eq__(self, other): def __repr__(self): return repr(self.value) + + +class DBValueSubString: + + def __init__(self, value): + self.value = str(value) + + def __eq__(self, other): + return str(other) in self.value + + def __repr__(self): + return repr(self.value) From 7d69ce84c75e39aa6bf26adfd45adf1d1c2807db Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:16:10 +0100 Subject: [PATCH 11/45] bdd: remove needs include index check Postgrsql versions without the feature are no longer supported. --- tests/bdd/environment.py | 7 ------- tests/bdd/flex/lua-index-definitions.feature | 4 ---- 2 files changed, 11 deletions(-) diff --git a/tests/bdd/environment.py b/tests/bdd/environment.py index 16919ec85..16afeeae4 100644 --- a/tests/bdd/environment.py +++ b/tests/bdd/environment.py @@ -154,10 +154,3 @@ def test_db(context, **kwargs): def working_directory(context, **kwargs): with tempfile.TemporaryDirectory() as tmpdir: yield Path(tmpdir) - - -def before_tag(context, tag): - if tag == 'needs-pg-index-includes': - if context.config.userdata['PG_VERSION'] < 110000: - context.scenario.skip("No index includes in PostgreSQL < 11") - diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature index 051253fa5..1400a7809 100644 --- a/tests/bdd/flex/lua-index-definitions.feature +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -255,7 +255,6 @@ Feature: Index definitions in Lua file | indexdef@substr | | USING btree (lower(name)) | - @needs-pg-index-includes Scenario: Include field must be a string or array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -279,7 +278,6 @@ Feature: Index definitions in Lua file The 'include' field in an index definition must contain a string or an array. """ - @needs-pg-index-includes Scenario: Include field must contain a valid column Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -303,7 +301,6 @@ Feature: Index definitions in Lua file Unknown column 'foo' in table 'mytable'. """ - @needs-pg-index-includes Scenario: Include field works with string Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -326,7 +323,6 @@ Feature: Index definitions in Lua file | indexdef@substr | | USING btree (name) INCLUDE (tags) | - @needs-pg-index-includes Scenario: Include field works with array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style From 69739e8dccce612ce0b9a8834aa839ed78ab130e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:52:05 +0100 Subject: [PATCH 12/45] bdd: replace remaining SELECT steps with SQL templates --- tests/bdd/flex/delete-callbacks.feature | 34 ++++++++++++++--------- tests/bdd/regression/multipolygon.feature | 28 +++++++++++++++---- tests/bdd/steps/steps_db.py | 7 +++-- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/tests/bdd/flex/delete-callbacks.feature b/tests/bdd/flex/delete-callbacks.feature index 2d2a3ec02..773c521db 100644 --- a/tests/bdd/flex/delete-callbacks.feature +++ b/tests/bdd/flex/delete-callbacks.feature @@ -27,21 +27,29 @@ Feature: Test for delete callbacks Given the input file '008-ch.osc.gz' + Given the SQL statement grouped_counts + """ + SELECT osm_type, + count(*), + sum(extra) AS sum_extra, + sum(osm_id) AS sum_osm_id + FROM change + GROUP BY osm_type + """ + Scenario: Delete callbacks are called When running osm2pgsql flex with parameters | --slim | -a | - - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | - | N | 16773 | 16779 | - | W | 4 | 9 | - | R | 1 | 3 | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | + | N | 16773 | 16779 | + | W | 4 | 9 | + | R | 1 | 3 | When running osm2pgsql flex with parameters | --slim | -a | - - Then SELECT osm_type, count(*), sum(osm_id) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_osm_id | | N | 16773 | 37856781001834 | | W | 4 | 350933407 | | R | 1 | 2871571 | @@ -72,8 +80,8 @@ Feature: Test for delete callbacks """ When running osm2pgsql flex with parameters | --slim | -a | - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | | N | 16773 | 16773 | | W | 4 | 4 | | R | 1 | 1 | @@ -105,8 +113,8 @@ Feature: Test for delete callbacks """ When running osm2pgsql flex with parameters | --slim | -a | - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | | N | 16773 | 16773| | W | 4 | 4| | R | 1 | 1| diff --git a/tests/bdd/regression/multipolygon.feature b/tests/bdd/regression/multipolygon.feature index ce4b48ef5..4f5d7af91 100644 --- a/tests/bdd/regression/multipolygon.feature +++ b/tests/bdd/regression/multipolygon.feature @@ -3,6 +3,16 @@ Feature: Import and update of multipolygon areas Background: Given the input file 'test_multipolygon.osm' + Given the SQL statement grouped_polygons + """ + SELECT osm_id, + count(*) AS count, + round(sum(ST_Area(way))) AS area, + round(sum(way_area::numeric)) AS way_area + FROM planet_osm_polygon + GROUP BY osm_id + """ + Scenario Outline: Import and update slim Given lua tagtransform When running osm2pgsql pgsql with parameters @@ -35,7 +45,7 @@ Feature: Import and update of multipolygon areas | osm_id | landuse | name | ST_NumInteriorRing(way) | | -3 | residential | Name_rel11 | 2 | - Then SELECT osm_id, round(sum(ST_Area(way))), round(sum(way_area::numeric)) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | area | way_area | | -13 | 17581 | 17581 | | -7 | 16169 | 16169 | @@ -43,7 +53,7 @@ Feature: Import and update of multipolygon areas | -39 | 10377 | 10378 | | -40 | 12397 | 12397 | - Then SELECT osm_id, count(*) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | count | | -25 | 1 | | 113 | 1 | @@ -63,7 +73,15 @@ Feature: Import and update of multipolygon areas | osm_id | "natural" | | -33 | water | - Then SELECT osm_id, CASE WHEN '' = '-G' THEN min(ST_NumGeometries(way)) ELSE count(*) END FROM planet_osm_polygon GROUP BY osm_id + Given the SQL statement geometries_polygon + """ + SELECT osm_id, + CASE WHEN '' = '-G' THEN min(ST_NumGeometries(way)) + ELSE count(*) END AS count + FROM planet_osm_polygon + GROUP BY osm_id + """ + Then statement geometries_polygon returns | osm_id | count | | -13 | 2 | | -7 | 2 | @@ -99,7 +117,7 @@ Feature: Import and update of multipolygon areas | osm_id | landuse | name | ST_NumInteriorRing(way) | | -3 | residential | Name_rel11 | 2 | - Then SELECT osm_id, round(sum(ST_Area(way))), round(sum(way_area::numeric)) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | area | way_area | | -13 | 17581 | 17581 | | -7 | 16169 | 16169 | @@ -107,7 +125,7 @@ Feature: Import and update of multipolygon areas | -39 | 10377 | 10378 | | -40 | 12397 | 12397 | - Then SELECT osm_id, count(*) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | count | | 113 | 1 | | 118 | 1 | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index c91d1e059..681b0635d 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -98,9 +98,12 @@ def db_define_sql_statement(context, sql): @then("statement (?P.+) returns(?P exactly)?") def db_check_sql_statement(context, stmt, exact): + assert stmt in context.sql_statements + rows = sql.SQL(', '.join(h.rsplit('@')[0] for h in context.table.headings)) + with context.db.cursor() as cur: - assert stmt in context.sql_statements - cur.execute(context.sql_statements[stmt]) + cur.execute(sql.SQL("SELECT {} FROM ({}) _sql_statement") + .format(rows, sql.SQL(context.sql_statements[stmt]))) actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur) From b2765aca4a272b2fd015f299c0557c22b86d2555 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 3 Nov 2025 09:31:11 +0100 Subject: [PATCH 13/45] introduce osm2pgsql-test-style for BDD testing --- scripts/osm2pgsql-test-style | 835 ++++++++++++++++++ tests/bdd/command-line/invalid.feature | 6 +- tests/bdd/command-line/replication.feature | 26 +- .../command-line/replication_legacy.feature | 17 +- tests/bdd/flex/area.feature | 6 +- tests/bdd/flex/bbox.feature | 18 +- tests/bdd/flex/delete-callbacks.feature | 6 +- tests/bdd/flex/empty-insert.feature | 4 +- tests/bdd/flex/geometry-collection.feature | 4 +- tests/bdd/flex/geometry-function-area.feature | 6 +- tests/bdd/flex/geometry-iteration.feature | 16 +- tests/bdd/flex/geometry-linestring.feature | 9 +- .../bdd/flex/geometry-multilinestring.feature | 8 +- tests/bdd/flex/geometry-multipoint.feature | 10 +- tests/bdd/flex/geometry-null.feature | 18 +- tests/bdd/flex/geometry-point.feature | 44 +- tests/bdd/flex/geometry-processing.feature | 22 +- tests/bdd/flex/invalid-geometries.feature | 10 +- tests/bdd/flex/invalid-lua.feature | 3 +- tests/bdd/flex/line-splitting.feature | 8 +- tests/bdd/flex/locator.feature | 36 +- .../lua-expire-output-definitions.feature | 27 +- tests/bdd/flex/lua-expire.feature | 18 +- tests/bdd/flex/lua-index-definitions.feature | 61 +- tests/bdd/flex/lua-table-definitions.feature | 21 +- tests/bdd/flex/multigeom.feature | 2 +- tests/bdd/flex/relation-changes.feature | 2 +- tests/bdd/flex/way-change.feature | 2 +- tests/bdd/regression/import.feature | 55 +- tests/bdd/regression/input.feature | 2 +- tests/bdd/regression/multipolygon.feature | 266 ++++-- tests/bdd/regression/properties.feature | 14 +- tests/bdd/regression/update.feature | 4 +- 33 files changed, 1314 insertions(+), 272 deletions(-) create mode 100755 scripts/osm2pgsql-test-style diff --git a/scripts/osm2pgsql-test-style b/scripts/osm2pgsql-test-style new file mode 100755 index 000000000..0cdb8bdd9 --- /dev/null +++ b/scripts/osm2pgsql-test-style @@ -0,0 +1,835 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: GPL-2.0-or-later +# +# This file is part of osm2pgsql (https://osm2pgsql.org/). +# +# Copyright (C) 2025 by the osm2pgsql developer community. +# For a full list of authors see the git log. +""" +Test runner for BDD-style integration tests. + +See osm2pgsql manual for more information on osm2pgsql style testing. +""" +import logging +import sys +import tempfile +import math +import re +import contextlib +import json +import datetime as dt +from decimal import Decimal +from subprocess import Popen, PIPE +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from pathlib import Path +import importlib.util +import io +from importlib.machinery import SourceFileLoader + +from behave import given, when, then, use_step_matcher, use_fixture, fixture +from behave.runner import ModelRunner, Context +from behave.formatter.base import StreamOpener +from behave.formatter.pretty import PrettyFormatter +from behave import runner_util +from behave.configuration import Configuration + +LOG = logging.getLogger() + +import psycopg +from psycopg import sql + +use_step_matcher('re') + +OBJECT_ORDER = {'n': 1, 'w': 2, 'r': 3} + +def opl_sort(line): + oid = line.split(' ', 1)[0] + return OBJECT_ORDER[oid[0]], int(oid[1:]) + +#################### Replication mock ############################## + +class ReplicationServerMock: + + def __init__(self, base_url, state_infos): + self.expected_base_url = base_url + self.state_infos = state_infos + + def __call__(self, base_url): + assert base_url == self.expected_base_url,\ + f"Wrong replication service called. Expected '{self.expected_base_url}', got '{base_url}'" + return self + + def get_state_info(self, seq=None, retries=2): + assert self.state_infos, 'Replication mock not properly set up' + if seq is None: + return self.state_infos[-1] + + for info in self.state_infos: + if info.sequence == seq: + return info + + return None + + def timestamp_to_sequence(self, timestamp, balanced_search=False): + assert self.state_infos, 'Replication mock not properly set up' + + if timestamp < self.state_infos[0].timestamp: + return self.state_infos[0].sequence + + prev = self.state_infos[0] + for info in self.state_infos: + if timestamp >= prev.timestamp and timestamp < info.timestamp: + return prev.sequence + prev = info + + return prev.sequence + + def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): + if start_id > self.state_infos[-1].sequence: + return None + + numdiffs = int((max_size + 1023)/1024) + return min(self.state_infos[-1].sequence, start_id + numdiffs - 1) + + +# Replication module is optional +_repfl_spec = importlib.util.spec_from_loader( + 'osm2pgsql_replication', + SourceFileLoader('osm2pgsql_replication', + str(Path(__file__, '..', 'osm2pgsql-replication').resolve()))) + +if _repfl_spec: + osm2pgsql_replication = importlib.util.module_from_spec(_repfl_spec) + _repfl_spec.loader.exec_module(osm2pgsql_replication) + + from osmium.replication.server import OsmosisState +else: + osm2pgsql_replication = None + +#################### hooks ######################################### + +def hook_before_all(context): + context.config.setup_logging(logging.INFO) + + # Feature check: table spaces + if context.user_args.test_tablespace == 'auto': + with context.connect_db('postgres') as conn: + with conn.cursor() as cur: + cur.execute("""SELECT spcname FROM pg_tablespace + WHERE spcname = 'tablespacetest'""") + context.user_args.test_tablespace = cur.rowcount > 0 + LOG.info('Check if tablespaces are available: %s', + 'yes' if context.user_args.test_tablespace else 'no') + else: + context.user_args.test_tablespace = context.user_args.test_tablespace == 'yes' + + # Test that osm2pgsql works. + proc = Popen([context.user_args.osm2pgsql_binary, '--version'], + stdout=PIPE, stderr=PIPE) + _, serr = proc.communicate() + osm2pgsql_version = serr.decode('utf-8') + if proc.returncode != 0: + LOG.critical("Could not run osm2pgsql. Error:\n%s", serr) + LOG.critical("osm2pgsql binary used: %s", context.user_args.osm2pgsql_binary) + raise RuntimeError('Error running osm2pgsql') + + LOG.info('Check if proj is available: %s', + 'yes' if context.user_args.test_proj else 'no') + + # Feature check: proj + if context.user_args.test_proj == 'auto': + context.user_args.test_proj = 'Proj [disabled]' not in osm2pgsql_version + else: + context.user_args.test_proj = context.user_args.test_proj == 'yes' + + use_fixture(template_test_db, context) + + + +def hook_before_scenario(context, scenario): + if 'config.have_proj' in scenario.tags and not context.user_args.test_proj: + scenario.skip("Generic proj library not configured.") + + context.db = use_fixture(test_db, context) + context.import_file = None + context.import_data = None + context.osm2pgsql_params = {'-d': context.user_args.test_db} + context.osm2pgsql_returncode = None + context.workdir = use_fixture(working_directory, context) + context.nodes = NodeStore() + context.sql_statements = {} + context.urlrequest_responses = {} + osm2pgsql_replication.ReplicationServer = None + + def _mock_urlopen(request): + if not request.full_url in context.urlrequest_responses: + raise urllib.error.URLError('Unknown URL') + + return contextlib.closing(io.BytesIO(context.urlrequest_responses[request.full_url].encode('utf-8'))) + + osm2pgsql_replication.urlrequest.urlopen = _mock_urlopen + + + +#################### fixtures ###################################### + +@fixture +def template_test_db(context, **kwargs): + context.drop_db(context.user_args.template_test_db, recreate_template='default') + + with context.connect_db(context.user_args.template_test_db) as conn: + conn.execute('CREATE EXTENSION postgis') + conn.execute('CREATE EXTENSION hstore') + + yield context.user_args.template_test_db + + context.drop_db(context.user_args.template_test_db) + + +@fixture +def test_db(context, **kwargs): + context.drop_db(recreate_template=context.user_args.template_test_db) + + with context.connect_db() as conn: + yield conn + + if not context.user_args.keep_test_db: + context.drop_db() + + +@fixture +def working_directory(context, **kwargs): + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + +################### Node location creation ######################### + +class NodeStore: + grid = {} + + def set_grid(self, lines, grid_step, origin_x, origin_y): + self.grid = {} + origin_y -= grid_step * (len(lines) - 1) + + ndigits = 1 + step = grid_step + while step < 0: + ndigits += 1 + step /= 10 + + for y, line in enumerate(lines): + for x, pt in enumerate(line): + if pt.isdigit(): + self.grid[int(pt)] = (round(origin_x + x * grid_step, ndigits), + round(origin_y + y * grid_step, ndigits)) + + def get_as_opl(self): + return [f"n{i} x{x} y{y}" for i, (x, y) in self.grid.items()] + + def add_coordinates(self, lines): + for line in lines: + if line.startswith('n') and ' x' not in line: + nid = int(line.split(' ', 1)[0][1:]) + assert nid in self.grid, \ + f"OPL error. Node {nid} has no coordinates and is not in grid." + x, y = self.grid[nid] + yield f"{line} x{x} y{y}" + else: + yield line + + def parse_point(self, pt): + pt = pt.strip() + if ' ' in pt: + return list(map(float, pt.split(' ', 1))) + return self.grid[int(pt)] + + +################### ResultComparison ############################### + +class ResultCompare: + + def __init__(self, heading, nodes): + self.nodes = nodes + if '!' in heading: + self.name, self.fmt = heading.rsplit('!', 1) + if self.fmt.startswith(':'): + self.compare = self._intcompare_fmt + elif self.fmt.startswith('~'): + if self.fmt.endswith('%'): + rel_tol = float(self.fmt[1:-1]) / 100 + self.compare = lambda exp, val: math.isclose(float(exp), val, rel_tol=rel_tol) + else: + abs_tol = float(self.fmt[1:]) + self.compare = lambda exp, val: math.isclose(float(exp), val, abs_tol=abs_tol) + else: + self.compare = getattr(self, f"_compare_{self.fmt}", None) + assert self.compare is not None, f"Unknown formatter {self.fmt}" + else: + self.name = heading + self.fmt = None + self.compare = lambda exp, val: str(val) == exp + + def as_select(self): + if self.fmt == 'geo': + return f"ST_AsText({self.name})" + + return self.name + + def equals(self, expected, value): + if expected == 'NULL': + return value is None + + return self.compare(expected, value) + + def _intcompare_fmt(self, expected, value): + return expected == f"{{{self.fmt}}}".format(value) + + def _compare_i(self, expected, value): + return expected.lower() == str(value).lower() + + def _compare_re(self, expected, value): + return re.fullmatch(expected, str(value)) is not None + + def _compare_substr(self, expected, value): + return expected in str(value) + + def _compare_json(self, expected, value): + return json.loads(expected) == value + + def _compare_geo(self, expected, value): + m = re.fullmatch(r'([A-Z]+)\((.*)\)', value) + + return self._eq_geom(expected, m[1], m[2]) if m else False + + def _eq_geom(self, bdd_geom, pg_type, pg_coords): + # MULTI* geometries + if bdd_geom.startswith('[') and bdd_geom.endswith(']'): + bdd_parts = bdd_geom[1:-1].split(';') + pg_parts = pg_coords[1:-1].split('),(') + return pg_type.startswith('MULTI') \ + and len(bdd_parts) == len(pg_parts) \ + and all(self._eq_geom(b.strip(), pg_type[5:], g.strip()) + for b, g in zip(bdd_parts, pg_parts)) + + # GEOMETRYCOLLECTIONS + if bdd_geom.startswith('{') and bdd_geom.endswith('}'): + bdd_parts = bdd_geom[1:-1].split(';') + pg_parts = list(map(lambda s: re.fullmatch(r'([A-Z]+)\(([^A-Z]*)\)', s), + re.findall('[A-Z]+[^A-Z]+[^,A-Z]', pg_coords))) + return pg_type.startswith('GEOMETRYCOLLECTION')\ + and len(bdd_parts) == len(pg_parts)\ + and all(g is not None and self._eq_geom(b.strip(), g[1], g[2]) + for b, g in zip(bdd_parts, pg_parts)) + + # POINT + if ',' not in bdd_geom: + return pg_type == 'POINT' and self._eq_point(bdd_geom, pg_coords) + + # LINESTRING + if '(' not in bdd_geom: + return pg_type == 'LINESTRING' \ + and all(self._eq_point(b, p) for b, p + in zip((g.strip() for g in bdd_geom.split(',')), + (g.strip() for g in pg_coords.split(',')))) + + # POLYGON + if pg_type != 'POLYGON': + return False + # Polygon comparison is tricky because the polygons don't necessarily + # end at the same point or have the same winding order. + # Brute force all possible variants of the expected polygon + bdd_parts = re.findall(r'\([^)]+\)', bdd_geom) + pg_parts = [g.strip() for g in pg_coords[1:-1].split('),(')] + return len(bdd_parts) == len(pg_parts) \ + and all(self._eq_ring(*parts) for parts in zip(bdd_parts, pg_parts)) + + def _eq_point(self, bdd_pt, pg_pt): + exp_geom = self.nodes.parse_point(bdd_pt) + pg_geom = list(map(float, pg_pt.split(' '))) + return len(exp_geom) == len(pg_geom) \ + and all(math.isclose(e, p, rel_tol=0.000001) for e, p in zip(exp_geom, pg_geom)) + + def _eq_ring(self, bdd_ring, pg_ring): + bdd_pts = [g.strip() for g in bdd_ring[1:-1].split(',')] + pg_pts = [g.strip() for g in pg_ring.split(',')] + if bdd_pts[0] != bdd_pts[-1]: + raise RuntimeError(f"Invalid polygon {bdd_geom}. " + "First and last point need to be the same") + if len(bdd_pts) != len(pg_pts): + return False + + for line in (bdd_pts[:-1], bdd_pts[-1:0:-1]): + for i in range(len(line)): + if all(self._eq_point(p1, p2) for p1, p2 in + zip(line[i:] + line[:i], pg_pts)): + return True + + return False + +################### Steps: Database setup ########################## + +@given("the database schema (?P.+)") +def create_db_schema(context, schema): + with context.db.cursor() as cur: + cur.execute("CREATE SCHEMA " + schema) + + +@when("deleting table (?P
.+)") +def delete_table(context, table): + with context.db.cursor() as cur: + cur.execute("DROP TABLE " + table) + + +################### Steps: OSM data ################################ + +@given("the input file '(?P.+)'") +def osm_set_import_file(context, osm_file): + assert context.import_data is None, \ + "Import file cannot be used together with inline data." + pfile = Path(osm_file) + if pfile.is_absolute(): + context.import_file = pfile + else: + basedir = context.user_args.test_data_dir or Path(context.feature.filename).parent + context.import_file = (basedir / osm_file).resolve() + + +@given("the OSM data") +def osm_define_data(context): + assert context.import_file is None, \ + "Inline data cannot be used together with an import file." + if context.text.strip(): + context.append_osm_data(context.nodes.add_coordinates(context.text.split('\n'))) + else: + context.append_osm_data([]) + + +@given("the OSM data format string") +def osm_define_data(context): + assert context.import_file is None, \ + "Inline data cannot be used together with an import file." + data = eval('f"""' + context.text + '"""') + + context.append_osm_data(context.nodes.add_coordinates(data.split('\n'))) + + +@given("the (?P[0-9.]+ )?grid(?: with origin (?P[0-9.-]+) (?P[0-9.-]+))?") +def osm_define_node_grid(context, step, origin_x, origin_y): + step = float(step.strip()) if step else 0.1 + x = float(origin_x) if origin_x else 20.0 + y = float(origin_y) if origin_y else 20.0 + + assert x > -180.0 and x < 180.0 + assert y > -90.0 and y < 90.0 + + context.nodes.set_grid([context.table.headings] + [list(h) for h in context.table], step, x, y) + context.append_osm_data(context.nodes.get_as_opl()) + +################### Steps: Style file ############################## + +@given("the style file '(?P