Skip to content

Commit c24c723

Browse files
jotojoto
committed
Store properties to a database table and retrieve for updates
Store command line settings in a database table called "osm2pgsql_properties" when importing data. This is mainly so that we can do updates without specifying those settings again (or checking that the settings match the ones used for import.) The table always has the same name and will be stored in the schema used for the middle tables (set with --middle-schema). Adds a new generic properties_t class which can store strings, integers and booleans as properties. Internally a map of strings is used for the properties, in the database a simple table with two text columns, property and value, is used. For boolean values strings 'true' and 'false' are used. Currently the following properties are stored in the database: - "attributes" (bool): -x/--extra-attributes setting - "flat_node_file": flat node file name - "prefix": prefix from --prefix - "updatable" (bool): can the database be updated, i.e. created with --slim and no --drop - "version": osm2pgsql version
1 parent e113df2 commit c24c723

11 files changed

Lines changed: 651 additions & 10 deletions

‎src/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ target_sources(osm2pgsql_lib PRIVATE
3131
pgsql-capabilities.cpp
3232
pgsql-helper.cpp
3333
progress-display.cpp
34+
properties.cpp
3435
reprojection.cpp
3536
table.cpp
3637
taginfo.cpp

‎src/options.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,7 @@ options_t::options_t(int argc, char *argv[]) : options_t()
510510
break;
511511
case 'p': // --prefix
512512
prefix = optarg;
513+
prefix_is_set = true;
513514
check_identifier(prefix, "--prefix parameter");
514515
break;
515516
case 'd': // --database

‎src/options.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ class options_t
9595
std::string conninfo; ///< connection info for database
9696

9797
std::string prefix{"planet_osm"}; ///< prefix for table names
98+
bool prefix_is_set = false;
9899

99100
/// Pg Tablespace to store indexes on main tables (no default TABLESPACE)
100101
std::string tblsmain_index{};

‎src/osm2pgsql.cpp‎

Lines changed: 147 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717
#include "pgsql.hpp"
1818
#include "pgsql-capabilities.hpp"
1919
#include "pgsql-helper.hpp"
20+
#include "properties.hpp"
2021
#include "util.hpp"
2122
#include "version.hpp"
2223

2324
#include <osmium/util/memory.hpp>
2425

26+
#include <boost/filesystem.hpp>
27+
2528
#include <exception>
2629
#include <memory>
2730
#include <utility>
@@ -78,25 +81,148 @@ static void run(options_t const &options)
7881
osmdata.stop();
7982
}
8083

81-
void check_db(options_t const &options)
84+
static void check_db(options_t const &options)
8285
{
8386
pg_conn_t const db_connection{options.conninfo};
8487

8588
init_database_capabilities(db_connection);
8689

8790
check_schema(options.middle_dbschema);
8891
check_schema(options.output_dbschema);
92+
}
93+
94+
// This is called in "create" mode to store properties into the database.
95+
static void store_properties(properties_t *properties, options_t const &options)
96+
{
97+
properties->set_bool("attributes", options.extra_attributes);
98+
99+
if (options.flat_node_file.empty()) {
100+
properties->set_string("flat_node_file", "");
101+
} else {
102+
properties->set_string(
103+
"flat_node_file",
104+
boost::filesystem::absolute(
105+
boost::filesystem::path{options.flat_node_file})
106+
.string());
107+
}
108+
109+
properties->set_string("prefix", options.prefix);
110+
properties->set_bool("updatable", options.slim && !options.droptemp);
111+
properties->set_string("version", get_osm2pgsql_short_version());
112+
113+
properties->store();
114+
}
115+
116+
static void check_updatable(properties_t const &properties)
117+
{
118+
if (properties.get_bool("updatable", false)) {
119+
return;
120+
}
121+
122+
throw std::runtime_error{
123+
"This database is not updatable. To create an"
124+
" updatable database use --slim (without --drop)."};
125+
}
89126

90-
// If we are in append mode and the middle nodes table isn't there,
91-
// it probably means we used a flat node store when we created this
92-
// database. Check for that and stop if it looks like we are missing
93-
// the node location store option.
94-
if (options.append && options.flat_node_file.empty()) {
95-
if (!has_table(options.middle_dbschema, options.prefix + "_nodes")) {
127+
static void check_attributes(properties_t const &properties, options_t *options)
128+
{
129+
bool const with_attributes = properties.get_bool("attributes", false);
130+
131+
if (options->extra_attributes) {
132+
if (!with_attributes) {
96133
throw std::runtime_error{
97-
"You seem to not have a nodes table. Did "
98-
"you forget the --flat-nodes option?"};
134+
"Can not update with attributes (-x/--extra-attributes)"
135+
" because original import was without attributes."};
99136
}
137+
return;
138+
}
139+
140+
if (with_attributes) {
141+
log_info("Updating with attributes (same as on import).");
142+
options->extra_attributes = true;
143+
}
144+
}
145+
146+
static void check_and_update_flat_node_file(properties_t *properties,
147+
options_t *options)
148+
{
149+
auto const flat_node_file_from_import =
150+
properties->get_string("flat_node_file", "");
151+
if (options->flat_node_file.empty()) {
152+
if (flat_node_file_from_import.empty()) {
153+
log_info("Not using flat node file (same as on import).");
154+
} else {
155+
options->flat_node_file = flat_node_file_from_import;
156+
log_info("Using flat node file '{}' (same as on import).",
157+
flat_node_file_from_import);
158+
}
159+
} else {
160+
const auto absolute_path =
161+
boost::filesystem::absolute(
162+
boost::filesystem::path{options->flat_node_file})
163+
.string();
164+
165+
if (flat_node_file_from_import.empty()) {
166+
throw fmt_error("Database was imported without flat node file. Can"
167+
" not use flat node file '{}' now.",
168+
options->flat_node_file);
169+
} else if (absolute_path == flat_node_file_from_import) {
170+
log_info("Using flat node file '{}' (same as on import).",
171+
flat_node_file_from_import);
172+
} else {
173+
log_info(
174+
"Using the flat node file you specified on the command line"
175+
" ('{}') instead of the one used on import ('{}').",
176+
absolute_path, flat_node_file_from_import);
177+
properties->set_string("flat_node_file", absolute_path, true);
178+
}
179+
}
180+
181+
}
182+
183+
static void check_prefix(properties_t const &properties, options_t *options)
184+
{
185+
auto const prefix = properties.get_string("prefix", "planet_osm");
186+
if (!options->prefix_is_set) {
187+
log_info("Using prefix '{}' (same as on import).", prefix);
188+
options->prefix = prefix;
189+
return;
190+
}
191+
192+
if (prefix != options->prefix) {
193+
throw fmt_error("Different prefix specified on command line ('{}')"
194+
" then used on import ('{}').",
195+
options->prefix, prefix);
196+
}
197+
}
198+
199+
// This is called in "append" mode to check that the command line options are
200+
// compatible with the properties stored in the database.
201+
static void check_and_update_properties(properties_t *properties,
202+
options_t *options)
203+
{
204+
check_updatable(*properties);
205+
check_attributes(*properties, options);
206+
check_and_update_flat_node_file(properties, options);
207+
check_prefix(*properties, options);
208+
}
209+
210+
// If we are in append mode and the middle nodes table isn't there, it probably
211+
// means we used a flat node store when we created this database. Check for
212+
// that and stop if it looks like we are missing the node location store
213+
// option. (This function is only used in legacy systems which don't have the
214+
// properties stored in the database.)
215+
static void check_for_nodes_table(options_t const &options)
216+
{
217+
if (!options.flat_node_file.empty()) {
218+
return;
219+
}
220+
221+
if (!has_table(options.middle_dbschema.empty() ? "public"
222+
: options.middle_dbschema,
223+
options.prefix + "_nodes")) {
224+
throw std::runtime_error{"You seem to not have a nodes table. Did "
225+
"you forget the --flat-nodes option?"};
100226
}
101227
}
102228

@@ -105,7 +231,7 @@ int main(int argc, char *argv[])
105231
try {
106232
log_info("osm2pgsql version {}", get_osm2pgsql_version());
107233

108-
options_t const options{argc, argv};
234+
options_t options{argc, argv};
109235
if (options.early_return()) {
110236
return 0;
111237
}
@@ -114,6 +240,17 @@ int main(int argc, char *argv[])
114240

115241
check_db(options);
116242

243+
properties_t properties{options.conninfo, options.middle_dbschema};
244+
if (options.append) {
245+
if (properties.load()) {
246+
check_and_update_properties(&properties, &options);
247+
} else {
248+
check_for_nodes_table(options);
249+
}
250+
} else {
251+
store_properties(&properties, options);
252+
}
253+
117254
run(options);
118255

119256
show_memory_usage();

‎src/pgsql-capabilities.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ static void init_set_from_query(std::set<std::string> *set,
3232
char const *table, char const *column,
3333
char const *condition = "true")
3434
{
35+
set->clear(); // Clear existing in case this is called multiple times
36+
3537
auto const res = db_connection.exec("SELECT {} FROM {} WHERE {}", column,
3638
table, condition);
3739
for (int i = 0; i < res.num_tuples(); ++i) {
@@ -42,6 +44,8 @@ static void init_set_from_query(std::set<std::string> *set,
4244
/// Get all config settings from the database.
4345
static void init_settings(pg_conn_t const &db_connection)
4446
{
47+
capabilities().settings.clear(); // In case this is called multiple times
48+
4549
auto const res =
4650
db_connection.exec("SELECT name, setting FROM pg_settings");
4751

0 commit comments

Comments
 (0)