Skip to content

Commit cb22845

Browse files
jburgess777jburgess777
committed
osm2pgsql: Fix relation processing in non-slim mode. It now needs more memory during the processing since it needs to remember ways even if they dont have any tags
1 parent 82b6531 commit cb22845

7 files changed

Lines changed: 72 additions & 49 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PACKAGE:=osm2pgsql
2-
VERSION:=0.55
2+
VERSION:=0.60
33
SVN:=$(shell date +%Y%m%d)
44

55
CC = gcc

‎middle-pgsql.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1201,24 +1201,26 @@ struct middle_t mid_pgsql = {
12011201
cleanup: pgsql_cleanup,
12021202
analyze: pgsql_analyze,
12031203
end: pgsql_end,
1204+
12041205
nodes_set: pgsql_nodes_set,
12051206
// nodes_get: pgsql_nodes_get,
12061207
nodes_get_list: pgsql_nodes_get_list,
12071208
nodes_delete: pgsql_nodes_delete,
12081209
node_changed: pgsql_node_changed,
1210+
12091211
ways_set: pgsql_ways_set,
12101212
ways_get: pgsql_ways_get,
12111213
ways_done: pgsql_ways_done,
12121214
ways_delete: pgsql_ways_delete,
12131215
way_changed: pgsql_way_changed,
1214-
// iterate_nodes: pgsql_iterate_nodes,
1215-
iterate_ways: pgsql_iterate_ways,
1216-
1216+
12171217
relations_set: pgsql_rels_set,
1218-
relations_get: pgsql_rels_get,
1218+
// relations_get: pgsql_rels_get,
12191219
relations_done: pgsql_rels_done,
12201220
relations_delete: pgsql_rels_delete,
12211221
relation_changed: pgsql_rel_changed,
1222-
1222+
1223+
// iterate_nodes: pgsql_iterate_nodes,
1224+
iterate_ways: pgsql_iterate_ways,
12231225
iterate_relations: pgsql_iterate_relations
12241226
};

‎middle-ram.c‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ struct ramNode {
4141
struct ramWay {
4242
struct keyval *tags;
4343
int *ndids;
44+
int pending;
4445
};
4546

4647
struct ramRel {
@@ -75,9 +76,7 @@ static int node_blocks;
7576
static int way_blocks;
7677

7778
static int way_out_count;
78-
#if 0
7979
static int rel_out_count;
80-
#endif
8180

8281
static inline int id2block(int id)
8382
{
@@ -148,10 +147,7 @@ static int ram_ways_set(int id, int *nds, int nd_count, struct keyval *tags, int
148147
int block = id2block(id);
149148
int offset = id2offset(id);
150149
struct keyval *p;
151-
152-
if( !pending ) /* If it's not still to be done, don't bother storing it at all */
153-
return 0;
154-
150+
155151
if (!ways[block]) {
156152
ways[block] = calloc(PER_BLOCK, sizeof(struct ramWay));
157153
if (!ways[block]) {
@@ -171,6 +167,7 @@ static int ram_ways_set(int id, int *nds, int nd_count, struct keyval *tags, int
171167
ways[block][offset].ndids = malloc( (nd_count+1)*sizeof(int) );
172168
memcpy( ways[block][offset].ndids+1, nds, nd_count*sizeof(int) );
173169
ways[block][offset].ndids[0] = nd_count;
170+
ways[block][offset].pending = pending;
174171

175172
if (!ways[block][offset].tags) {
176173
p = malloc(sizeof(struct keyval));
@@ -184,8 +181,7 @@ static int ram_ways_set(int id, int *nds, int nd_count, struct keyval *tags, int
184181
} else
185182
resetList(ways[block][offset].tags);
186183

187-
while ((p = popItem(tags)) != NULL)
188-
pushItem(ways[block][offset].tags, p);
184+
cloneList(ways[block][offset].tags, tags);
189185

190186
return 0;
191187
}
@@ -215,8 +211,7 @@ static int ram_relations_set(int id, struct member *members, int member_count, s
215211
} else
216212
resetList(rels[block][offset].tags);
217213

218-
while ((p = popItem(tags)) != NULL)
219-
pushItem(rels[block][offset].tags, p);
214+
cloneList(rels[block][offset].tags, tags);
220215

221216
if (!rels[block][offset].members)
222217
free( rels[block][offset].members );
@@ -232,7 +227,6 @@ static int ram_relations_set(int id, struct member *members, int member_count, s
232227
}
233228

234229
return 0;
235-
//return out_pgsql.relation(id, tags, nodes, ndCount)
236230
}
237231

238232
static int ram_nodes_get_list(struct osmNode *nodes, int *ndids, int nd_count)
@@ -250,9 +244,35 @@ static int ram_nodes_get_list(struct osmNode *nodes, int *ndids, int nd_count)
250244
return count;
251245
}
252246

253-
static void ram_iterate_relations(int (*callback)(int id, struct member *members, int member_count, struct keyval *tags, int) __unused)
247+
static void ram_iterate_relations(int (*callback)(int id, struct member *members, int member_count, struct keyval *tags, int))
254248
{
255-
/* Void */
249+
int block, offset;
250+
251+
fprintf(stderr, "\n");
252+
for(block=NUM_BLOCKS-1; block>=0; block--) {
253+
if (!rels[block])
254+
continue;
255+
256+
for (offset=0; offset < PER_BLOCK; offset++) {
257+
if (rels[block][offset].members) {
258+
int id = block2id(block, offset);
259+
rel_out_count++;
260+
if (rel_out_count % 1000 == 0)
261+
fprintf(stderr, "\rWriting rel(%uk)", rel_out_count/1000);
262+
263+
callback(id, rels[block][offset].members, rels[block][offset].member_count, rels[block][offset].tags, 0);
264+
}
265+
free(rels[block][offset].members);
266+
rels[block][offset].members = NULL;
267+
resetList(rels[block][offset].tags);
268+
free(rels[block][offset].tags);
269+
rels[block][offset].tags=NULL;
270+
}
271+
free(rels[block]);
272+
rels[block] = NULL;
273+
}
274+
275+
fprintf(stderr, "\rWriting rel(%uk)\n", rel_out_count/1000);
256276
}
257277

258278
static void ram_iterate_ways(int (*callback)(int id, struct keyval *tags, struct osmNode *nodes, int count, int exists))
@@ -271,18 +291,18 @@ static void ram_iterate_ways(int (*callback)(int id, struct keyval *tags, struct
271291
if (way_out_count % 1000 == 0)
272292
fprintf(stderr, "\rWriting way(%uk)", way_out_count/1000);
273293

274-
if (ways[block][offset].tags)
275-
if( getItem( ways[block][offset].tags, "multipolygon" ) )
276-
continue;
277-
278-
/* First element contains number of nodes */
279-
nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]);
280-
ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]);
281-
282-
if (nodes) {
283-
int id = block2id(block, offset);
284-
callback(id, ways[block][offset].tags, nodes, ndCount, 0);
285-
free(nodes);
294+
if (ways[block][offset].pending) {
295+
/* First element contains number of nodes */
296+
nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]);
297+
ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]);
298+
299+
if (nodes) {
300+
int id = block2id(block, offset);
301+
callback(id, ways[block][offset].tags, nodes, ndCount, 0);
302+
free(nodes);
303+
}
304+
305+
ways[block][offset].pending = 0;
286306
}
287307

288308
if (ways[block][offset].tags) {
@@ -324,17 +344,15 @@ static int ram_ways_get( int id, struct keyval *tags_ptr, struct osmNode **nodes
324344
return 1;
325345
}
326346

327-
// Makrs the way so that iterate ways skips it
347+
// Marks the way so that iterate ways skips it
328348
static int ram_ways_done( int id )
329349
{
330350
int block = id2block(id), offset = id2offset(id);
331351

332352
if (!ways[block])
333353
return 1;
334354

335-
if( ways[block][offset].tags )
336-
addItem( ways[block][offset].tags, "multipolygon", "1", 1 );
337-
355+
ways[block][offset].pending = 0;
338356
return 0;
339357
}
340358

@@ -394,10 +412,13 @@ struct middle_t mid_ram = {
394412
// nodes_get: ram_nodes_get,
395413
nodes_get_list: ram_nodes_get_list,
396414
ways_set: ram_ways_set,
397-
relations_set: ram_relations_set,
398415
ways_get: ram_ways_get,
399416
ways_done: ram_ways_done,
400-
// iterate_nodes: ram_iterate_nodes,
417+
418+
relations_set: ram_relations_set,
419+
420+
// iterate_nodes: ram_iterate_nodes,
401421
iterate_ways: ram_iterate_ways,
402422
iterate_relations: ram_iterate_relations
403423
};
424+

‎middle.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,25 @@ struct middle_t {
1717
void (*cleanup)(void);
1818
void (*analyze)(void);
1919
void (*end)(void);
20+
2021
int (*nodes_set)(int id, double lat, double lon, struct keyval *tags);
2122
int (*nodes_get_list)(struct osmNode *out, int *nds, int nd_count);
2223
int (*nodes_delete)(int id);
2324
int (*node_changed)(int id);
2425
// int (*nodes_get)(struct osmNode *out, int id);
26+
2527
int (*ways_set)(int id, int *nds, int nd_count, struct keyval *tags, int pending);
2628
int (*ways_get)(int id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr);
2729
int (*ways_done)(int id);
2830
int (*ways_delete)(int id);
2931
int (*way_changed)(int id);
32+
3033
int (*relations_set)(int id, struct member *members, int member_count, struct keyval *tags);
31-
int (*relations_get)(int id, struct member **members, int *member_count, struct keyval *tags);
34+
// int (*relations_get)(int id, struct member **members, int *member_count, struct keyval *tags);
3235
int (*relations_done)(int id);
3336
int (*relations_delete)(int id);
3437
int (*relation_changed)(int id);
38+
3539
// void (*iterate_nodes)(int (*callback)(int id, struct keyval *tags, double node_lat, double node_lon));
3640
void (*iterate_ways)(int (*callback)(int id, struct keyval *tags, struct osmNode *nodes, int count, int exists));
3741
void (*iterate_relations)(int (*callback)(int id, struct member *, int member_count, struct keyval *rel_tags, int exists));

‎osm2pgsql.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
291291
action = ACTION_DELETE;
292292
} else if (xmlStrEqual(name, BAD_CAST "bound")) {
293293
/* ignore */
294+
} else if (xmlStrEqual(name, BAD_CAST "bounds")) {
295+
/* ignore */
294296
} else {
295297
fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
296298
}
@@ -365,6 +367,8 @@ void EndElement(const xmlChar *name)
365367
filetype = FILETYPE_NONE;
366368
} else if (xmlStrEqual(name, BAD_CAST "bound")) {
367369
/* ignore */
370+
} else if (xmlStrEqual(name, BAD_CAST "bounds")) {
371+
/* ignore */
368372
} else if (xmlStrEqual(name, BAD_CAST "add")) {
369373
action = ACTION_NONE;
370374
} else if (xmlStrEqual(name, BAD_CAST "create")) {

‎output-pgsql.c‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,11 +1058,6 @@ static int pgsql_add_way(int id, int *nds, int nd_count, struct keyval *tags)
10581058
// Check whether the way is: (1) Exportable, (2) Maybe a polygon
10591059
int filter = pgsql_filter_tags(OSMTYPE_WAY, tags, &polygon);
10601060

1061-
// Memory saving hack:-
1062-
// If we're not in slim mode and it's not wanted, we can quit right away */
1063-
if( !Options->slim && filter )
1064-
return 1;
1065-
10661061
// If this isn't a polygon then it can not be part of a multipolygon
10671062
// Hence only polygons are "pending"
10681063
Options->mid->ways_set(id, nds, nd_count, tags, (!filter && polygon) ? 1 : 0);

‎output-pgsql.h‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
/* Implements the mid-layer processing for osm2pgsql
2-
* using several PostgreSQL tables
3-
*
4-
* This layer stores data read in from the planet.osm file
5-
* and is then read by the backend processing code to
6-
* emit the final geometry-enabled output formats
1+
/* Implements the output-layer processing for osm2pgsql
2+
* storing the data in several PostgreSQL tables
3+
* with the final PostGIS geometries for each entity
74
*/
85

96
#ifndef OUTPUT_PGSQL_H

0 commit comments

Comments
 (0)