Skip to content

Commit 0745459

Browse files
jotojoto
committed
Add properties from input file(s) to database
This adds two new properties to the database: "import_timestamp" and "current_timestamp". Both initially contain the highest timestamp found on any of the objects in the input file(s). The "import_timestamp" will never be changed again, the "current_timestamp" will be updated in append mode. If the input file(s) don't contain timestamps, the properties will not be created. It also adds properties "replication_base_url", "replication_sequence_number", and "replication_timestamp" if there is only a single input file and if that file has the respective header fields set.
1 parent 46c5cb7 commit 0745459

File tree

6 files changed

+233
-20
lines changed

6 files changed

+233
-20
lines changed

‎src/input.cpp‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -261,29 +261,40 @@ class input_context_t
261261
bool m_append;
262262
}; // class input_context_t
263263

264-
static void process_single_file(osmium::io::File const &file,
265-
osmdata_t *osmdata,
266-
progress_display_t *progress, bool append)
264+
static file_info process_single_file(osmium::io::File const &file,
265+
osmdata_t *osmdata,
266+
progress_display_t *progress, bool append)
267267
{
268+
file_info finfo;
269+
268270
osmium::io::Reader reader{file};
271+
finfo.header = reader.header();
269272
type_id last{osmium::item_type::node, 0};
270273

271274
input_context_t ctx{osmdata, progress, append};
272275
while (osmium::memory::Buffer buffer = reader.read()) {
273276
for (auto &object : buffer.select<osmium::OSMObject>()) {
274277
last = check_input(last, object);
275278
ctx.apply(&object);
279+
if (object.timestamp() > finfo.last_timestamp) {
280+
finfo.last_timestamp = object.timestamp();
281+
}
276282
}
277283
}
278284
ctx.eof();
279285

280286
reader.close();
287+
288+
return finfo;
281289
}
282290

283-
static void process_multiple_files(std::vector<osmium::io::File> const &files,
284-
osmdata_t *osmdata,
285-
progress_display_t *progress, bool append)
291+
static file_info
292+
process_multiple_files(std::vector<osmium::io::File> const &files,
293+
osmdata_t *osmdata, progress_display_t *progress,
294+
bool append)
286295
{
296+
file_info finfo;
297+
287298
std::vector<data_source_t> data_sources;
288299
data_sources.reserve(files.size());
289300

@@ -303,6 +314,9 @@ static void process_multiple_files(std::vector<osmium::io::File> const &files,
303314
queue.pop();
304315
if (queue.empty() || element != queue.top()) {
305316
ctx.apply(&element.object());
317+
if (element.object().timestamp() > finfo.last_timestamp) {
318+
finfo.last_timestamp = element.object().timestamp();
319+
}
306320
}
307321

308322
auto *source = element.data_source();
@@ -315,18 +329,20 @@ static void process_multiple_files(std::vector<osmium::io::File> const &files,
315329
for (auto &data_source : data_sources) {
316330
data_source.close();
317331
}
332+
333+
return finfo;
318334
}
319335

320-
void process_files(std::vector<osmium::io::File> const &files,
321-
osmdata_t *osmdata, bool append, bool show_progress)
336+
file_info process_files(std::vector<osmium::io::File> const &files,
337+
osmdata_t *osmdata, bool append, bool show_progress)
322338
{
323339
assert(osmdata);
324340

325341
progress_display_t progress{show_progress};
326342

327343
if (files.size() == 1) {
328-
process_single_file(files.front(), osmdata, &progress, append);
329-
} else {
330-
process_multiple_files(files, osmdata, &progress, append);
344+
return process_single_file(files.front(), osmdata, &progress, append);
331345
}
346+
347+
return process_multiple_files(files, osmdata, &progress, append);
332348
}

‎src/input.hpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include <osmium/fwd.hpp>
2323
#include <osmium/io/file.hpp>
24+
#include <osmium/io/header.hpp>
2425

2526
#include "osmtypes.hpp"
2627

@@ -32,6 +33,12 @@ struct type_id
3233
osmid_t id;
3334
};
3435

36+
struct file_info
37+
{
38+
osmium::io::Header header{};
39+
osmium::Timestamp last_timestamp{};
40+
};
41+
3542
/**
3643
* Compare two tuples (type, id). Throw a descriptive error if either the
3744
* curr id is negative or if the data is not ordered.
@@ -51,7 +58,7 @@ prepare_input_files(std::vector<std::string> const &input_files,
5158
/**
5259
* Process the specified OSM files (stage 1a).
5360
*/
54-
void process_files(std::vector<osmium::io::File> const &files,
55-
osmdata_t *osmdata, bool append, bool show_progress);
61+
file_info process_files(std::vector<osmium::io::File> const &files,
62+
osmdata_t *osmdata, bool append, bool show_progress);
5663

5764
#endif // OSM2PGSQL_INPUT_HPP

‎src/osm2pgsql.cpp‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ static void show_memory_usage()
4343
}
4444
}
4545

46-
static void run(options_t const &options)
46+
static file_info run(options_t const &options)
4747
{
4848
auto const files = prepare_input_files(
4949
options.input_files, options.input_format, options.append);
@@ -71,14 +71,16 @@ static void run(options_t const &options)
7171

7272
// Processing: In this phase the input file(s) are read and parsed,
7373
// populating some of the tables.
74-
process_files(files, &osmdata, options.append,
75-
get_logger().show_progress());
74+
auto const finfo = process_files(files, &osmdata, options.append,
75+
get_logger().show_progress());
7676

7777
show_memory_usage();
7878

7979
// Process pending ways and relations. Cluster database tables and
8080
// create indexes.
8181
osmdata.stop();
82+
83+
return finfo;
8284
}
8385

8486
static void check_db(options_t const &options)
@@ -113,6 +115,25 @@ static void store_properties(properties_t *properties, options_t const &options)
113115
properties->store();
114116
}
115117

118+
static void store_data_properties(properties_t *properties,
119+
file_info const &finfo)
120+
{
121+
if (finfo.last_timestamp.valid()) {
122+
auto const timestamp = finfo.last_timestamp.to_iso();
123+
properties->set_string("import_timestamp", timestamp);
124+
properties->set_string("current_timestamp", timestamp);
125+
}
126+
127+
for (std::string const s : {"base_url", "sequence_number", "timestamp"}) {
128+
auto const value = finfo.header.get("osmosis_replication_" + s);
129+
if (!value.empty()) {
130+
properties->set_string("replication_" + s, value);
131+
}
132+
}
133+
134+
properties->store();
135+
}
136+
116137
static void check_updatable(properties_t const &properties)
117138
{
118139
if (properties.get_bool("updatable", false)) {
@@ -247,12 +268,26 @@ int main(int argc, char *argv[])
247268
} else {
248269
check_for_nodes_table(options);
249270
}
271+
272+
auto const finfo = run(options);
273+
274+
if (finfo.last_timestamp.valid()) {
275+
auto const current_timestamp =
276+
properties.get_string("current_timestamp", "");
277+
278+
if (current_timestamp.empty() ||
279+
(finfo.last_timestamp >
280+
osmium::Timestamp{current_timestamp})) {
281+
properties.set_string("current_timestamp",
282+
finfo.last_timestamp.to_iso(), true);
283+
}
284+
}
250285
} else {
251286
store_properties(&properties, options);
287+
auto const finfo = run(options);
288+
store_data_properties(&properties, finfo);
252289
}
253290

254-
run(options);
255-
256291
show_memory_usage();
257292
log_info("osm2pgsql took {} overall.",
258293
util::human_readable_duration(timer_overall.stop()));

‎src/properties.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ void properties_t::set_string(std::string property, std::string value,
8282
auto const r =
8383
m_properties.insert_or_assign(std::move(property), std::move(value));
8484

85-
if (!update_database) {
85+
if (!update_database || !m_has_properties_table) {
8686
return;
8787
}
8888

‎tests/bdd/regression/timestamps.feature‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
Feature: Timestamps in properties table should reflect timestamps in input file
2+
3+
Scenario: Create database with timestamps
4+
Given the OSM data
5+
"""
6+
n10 t2020-01-02T03:04:05Z x10 y10
7+
n11 t2020-01-02T03:04:05Z x10 y11
8+
w20 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11
9+
"""
10+
When running osm2pgsql pgsql
11+
12+
Then table osm2pgsql_properties has 7 rows
13+
And table osm2pgsql_properties contains
14+
| property | value |
15+
| attributes | false |
16+
| flat_node_file | |
17+
| prefix | planet_osm |
18+
| updatable | false |
19+
| import_timestamp | 2020-01-02T03:04:06Z |
20+
| current_timestamp | 2020-01-02T03:04:06Z |
21+
22+
Scenario: Create database without timestamps
23+
Given the OSM data
24+
"""
25+
n10 x10 y10
26+
n11 x10 y11
27+
w20 Thighway=primary Nn10,n11
28+
"""
29+
When running osm2pgsql pgsql
30+
31+
Then table osm2pgsql_properties has 5 rows
32+
Then table osm2pgsql_properties contains
33+
| property | value |
34+
| attributes | false |
35+
| flat_node_file | |
36+
| prefix | planet_osm |
37+
| updatable | false |
38+
39+
Scenario: Create and update database with timestamps
40+
Given the OSM data
41+
"""
42+
n10 v1 t2020-01-02T03:04:05Z x10 y10
43+
n11 v1 t2020-01-02T03:04:05Z x10 y11
44+
w20 v1 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11
45+
"""
46+
47+
When running osm2pgsql pgsql with parameters
48+
| --create | --slim |
49+
50+
Then table osm2pgsql_properties has 7 rows
51+
And table osm2pgsql_properties contains
52+
| property | value |
53+
| attributes | false |
54+
| flat_node_file | |
55+
| prefix | planet_osm |
56+
| updatable | true |
57+
| import_timestamp | 2020-01-02T03:04:06Z |
58+
| current_timestamp | 2020-01-02T03:04:06Z |
59+
60+
Given the OSM data
61+
"""
62+
n11 v2 t2020-01-02T03:06:05Z x10 y12
63+
w20 v2 t2020-01-02T03:05:06Z Thighway=secondary Nn10,n11
64+
"""
65+
66+
When running osm2pgsql pgsql with parameters
67+
| --append | --slim |
68+
69+
Then table osm2pgsql_properties has 7 rows
70+
And table osm2pgsql_properties contains
71+
| property | value |
72+
| attributes | false |
73+
| flat_node_file | |
74+
| prefix | planet_osm |
75+
| updatable | true |
76+
| import_timestamp | 2020-01-02T03:04:06Z |
77+
| current_timestamp | 2020-01-02T03:06:05Z |
78+
79+
Scenario: Create database with timestamps and update without timestamps
80+
Given the OSM data
81+
"""
82+
n10 v1 t2020-01-02T03:04:05Z x10 y10
83+
n11 v1 t2020-01-02T03:04:05Z x10 y11
84+
w20 v1 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11
85+
"""
86+
87+
When running osm2pgsql pgsql with parameters
88+
| --create | --slim |
89+
90+
Then table osm2pgsql_properties has 7 rows
91+
And table osm2pgsql_properties contains
92+
| property | value |
93+
| attributes | false |
94+
| flat_node_file | |
95+
| prefix | planet_osm |
96+
| updatable | true |
97+
| import_timestamp | 2020-01-02T03:04:06Z |
98+
| current_timestamp | 2020-01-02T03:04:06Z |
99+
100+
Given the OSM data
101+
"""
102+
n11 v2 x10 y12
103+
w20 v2 Thighway=secondary Nn10,n11
104+
"""
105+
106+
When running osm2pgsql pgsql with parameters
107+
| --append | --slim |
108+
109+
Then table osm2pgsql_properties has 7 rows
110+
And table osm2pgsql_properties contains
111+
| property | value |
112+
| attributes | false |
113+
| flat_node_file | |
114+
| prefix | planet_osm |
115+
| updatable | true |
116+
| import_timestamp | 2020-01-02T03:04:06Z |
117+
| current_timestamp | 2020-01-02T03:04:06Z |
118+
119+
Scenario: Create database without timestamps and update with timestamps
120+
Given the OSM data
121+
"""
122+
n10 v1 x10 y10
123+
n11 v1 x10 y11
124+
w20 v1 Thighway=primary Nn10,n11
125+
"""
126+
127+
When running osm2pgsql pgsql with parameters
128+
| --create | --slim |
129+
130+
Then table osm2pgsql_properties has 5 rows
131+
And table osm2pgsql_properties contains
132+
| property | value |
133+
| attributes | false |
134+
| flat_node_file | |
135+
| prefix | planet_osm |
136+
| updatable | true |
137+
138+
Given the OSM data
139+
"""
140+
n11 v2 t2020-01-02T03:06:05Z x10 y12
141+
w20 v2 t2020-01-02T03:05:06Z Thighway=secondary Nn10,n11
142+
"""
143+
144+
When running osm2pgsql pgsql with parameters
145+
| --append | --slim |
146+
147+
Then table osm2pgsql_properties has 6 rows
148+
And table osm2pgsql_properties contains
149+
| property | value |
150+
| attributes | false |
151+
| flat_node_file | |
152+
| prefix | planet_osm |
153+
| updatable | true |
154+
| current_timestamp | 2020-01-02T03:06:05Z |
155+

‎tests/bdd/regression/update.feature‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Feature: Updates to the test database
2222
And table planet_osm_line has 3274 rows
2323
And table planet_osm_roads has 380 rows
2424
And table planet_osm_polygon has 4277 rows
25-
And table osm2pgsql_properties has 5 rows
25+
And table osm2pgsql_properties has 10 rows
2626

2727
Examples:
2828
| param1 | param2 | param3 |

0 commit comments

Comments
 (0)