From 008c6718582ebe64d24072d146e36ffad6265616 Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Tue, 9 Nov 2010 22:58:23 -0800 Subject: [PATCH 1/7] having two cursors open at the same time does some bizarre things to graph compilation. this fixes that. --- pygs/graphserver/compiler/gdb_import_gtfs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pygs/graphserver/compiler/gdb_import_gtfs.py b/pygs/graphserver/compiler/gdb_import_gtfs.py index 58c3e435..f91458f6 100644 --- a/pygs/graphserver/compiler/gdb_import_gtfs.py +++ b/pygs/graphserver/compiler/gdb_import_gtfs.py @@ -219,17 +219,15 @@ def gdb_load_gtfsdb(gdb, agency_namespace, gtfsdb, cursor, agency_id=None, maxtr acceptable_service_ids = None compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter ) - c = gdb.get_cursor() v_added = set([]) for fromv_label, tov_label, edge in compiler.gtfsdb_to_edges( maxtrips, service_ids=acceptable_service_ids ): if fromv_label not in v_added: - gdb.add_vertex( fromv_label, c ) + gdb.add_vertex( fromv_label, cursor ) v_added.add(fromv_label) if tov_label not in v_added: - gdb.add_vertex( tov_label, c ) + gdb.add_vertex( tov_label, cursor ) v_added.add(tov_label) - gdb.add_edge( fromv_label, tov_label, edge, c ) - gdb.commit() + gdb.add_edge( fromv_label, tov_label, edge, cursor ) def graph_load_gtfsdb( agency_namespace, gtfsdb, agency_id=None, maxtrips=None, reporter=sys.stdout ): compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter ) From c08264de13f1d43ca99f61599b95f2a0433ad37c Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Tue, 9 Nov 2010 22:59:04 -0800 Subject: [PATCH 2/7] prevent duplicate vertices --- pygs/graphserver/graphdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygs/graphserver/graphdb.py b/pygs/graphserver/graphdb.py index 62b215fd..99ca5f79 100644 --- a/pygs/graphserver/graphdb.py +++ b/pygs/graphserver/graphdb.py @@ -25,7 +25,7 @@ def __init__(self, sqlite_filename, overwrite=False): def setup(self): c = self.conn.cursor() - c.execute( "CREATE TABLE vertices (label)" ) + c.execute( "CREATE TABLE vertices (label TEXT UNIQUE ON CONFLICT IGNORE)" ) c.execute( "CREATE TABLE payloads (id TEXT UNIQUE ON CONFLICT IGNORE, type TEXT, state TEXT)" ) c.execute( "CREATE TABLE edges (vertex1 TEXT, vertex2 TEXT, epid TEXT)" ) c.execute( "CREATE TABLE resources (name TEXT UNIQUE ON CONFLICT IGNORE, image TEXT)" ) From bdb425547f061a504dbbe44418fe72c0b92b464a Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Wed, 10 Nov 2010 01:10:16 -0800 Subject: [PATCH 3/7] use the payload table auto-increment to assign ids, not memory pointers --- pygs/graphserver/graphdb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pygs/graphserver/graphdb.py b/pygs/graphserver/graphdb.py index 99ca5f79..6b9c34f0 100644 --- a/pygs/graphserver/graphdb.py +++ b/pygs/graphserver/graphdb.py @@ -26,8 +26,8 @@ def __init__(self, sqlite_filename, overwrite=False): def setup(self): c = self.conn.cursor() c.execute( "CREATE TABLE vertices (label TEXT UNIQUE ON CONFLICT IGNORE)" ) - c.execute( "CREATE TABLE payloads (id TEXT UNIQUE ON CONFLICT IGNORE, type TEXT, state TEXT)" ) - c.execute( "CREATE TABLE edges (vertex1 TEXT, vertex2 TEXT, epid TEXT)" ) + c.execute( "CREATE TABLE payloads (id INTEGER PRIMARY KEY, type TEXT, state TEXT)" ) + c.execute( "CREATE TABLE edges (vertex1 TEXT, vertex2 TEXT, epid INTEGER)" ) c.execute( "CREATE TABLE resources (name TEXT UNIQUE ON CONFLICT IGNORE, image TEXT)" ) self.conn.commit() @@ -39,9 +39,9 @@ def put_edge_payload(self, edgepayload, cc): for component in edgepayload.components: self.put_edge_payload( component, cc ) - cc.execute( "INSERT INTO payloads VALUES (?, ?, ?)", ( str(edgepayload.soul), cPickle.dumps( edgepayload.__class__ ), cPickle.dumps( edgepayload.__getstate__() ) ) ) - - return str(edgepayload.soul) + cc.execute( "INSERT INTO payloads (type, state) VALUES (?, ?)", ( cPickle.dumps( edgepayload.__class__ ), cPickle.dumps( edgepayload.__getstate__() ) ) ) + + return cc.lastrowid def get_edge_payload(self, id): queryresult = list(self.execute( "SELECT id, type, state FROM payloads WHERE id=?", (id,) )) From 2e10d2cea60dd6f1e24c9c1b1d06053279aff373 Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Wed, 1 Dec 2010 11:08:21 -0800 Subject: [PATCH 4/7] fixed memory leak --- pygs/graphserver/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pygs/graphserver/core.py b/pygs/graphserver/core.py index 626f64e0..185a57ea 100644 --- a/pygs/graphserver/core.py +++ b/pygs/graphserver/core.py @@ -323,6 +323,8 @@ def vertices(self): for i in range(count.value): v = SPTVertex.from_pointer(arr[i]) verts.append(v) + del arr + libc.free(p_va) return verts @property From de6b9aade3b8ab74c18e80d1a918e7e5279614ab Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Tue, 22 Feb 2011 23:45:29 -0800 Subject: [PATCH 5/7] moved Timezone struct definition from ServiceCalendar to Timezone --- core/servicecalendar.h | 4 ---- core/timezone.h | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/servicecalendar.h b/core/servicecalendar.h index 34dea300..4526a43b 100644 --- a/core/servicecalendar.h +++ b/core/servicecalendar.h @@ -82,8 +82,4 @@ spPrintPeriod( ServicePeriod* this ) ; inline long spNormalizeTime( ServicePeriod* this, int timezone_offset, long time ) ; -struct Timezone { - TimezonePeriod* head; -} ; - #endif diff --git a/core/timezone.h b/core/timezone.h index 3b5bd53b..9d1febde 100644 --- a/core/timezone.h +++ b/core/timezone.h @@ -8,6 +8,10 @@ struct TimezonePeriod { TimezonePeriod* next_period; } ; +struct Timezone { + TimezonePeriod* head; +} ; + Timezone* tzNew(void); From 59f9c48c140bab3faa84cf46b38a101db707af7f Mon Sep 17 00:00:00 2001 From: David Zwarg Date: Mon, 3 Jan 2011 18:00:30 -0500 Subject: [PATCH 6/7] Added fix for divide-by-zero bug, which shows up as "Negative weight (osm-307690545(4269609) -> osm-307690545(-2143214080))" during building of shortest path trees. --- core/edgetypes/street.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/edgetypes/street.c b/core/edgetypes/street.c index 3fcaf280..c84cafed 100644 --- a/core/edgetypes/street.c +++ b/core/edgetypes/street.c @@ -109,7 +109,10 @@ streetWalkGeneral(EdgePayload* superthis, State* state, WalkOptions* options, in Street* this = (Street*)superthis; State* ret = stateDup( state ); - float average_grade = (this->rise-this->fall)/this->length; + float average_grade = 0; + if (this->length > 0) { + average_grade = (this->rise-this->fall)/this->length; + } float average_speed = speed_from_grade(options, average_grade); long delta_t = this->length / average_speed; From 1fdce42a747df35a54ed4fa364837fb73710c226 Mon Sep 17 00:00:00 2001 From: Brandon Martin-Anderson Date: Fri, 25 Feb 2011 12:35:41 -0800 Subject: [PATCH 7/7] fixed bug recently introduced preventing Combinations in the graphdb --- pygs/graphserver/core.py | 3 +-- pygs/graphserver/graphdb.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pygs/graphserver/core.py b/pygs/graphserver/core.py index 185a57ea..76cd2b49 100644 --- a/pygs/graphserver/core.py +++ b/pygs/graphserver/core.py @@ -1491,7 +1491,6 @@ def __repr__(self): return ""%list(self.get_all_crossings()) class Combination(EdgePayload): - n = cproperty( lgs.comboN, c_int ) def __init__(self, cap): @@ -1508,7 +1507,7 @@ def to_xml(self): return ""%self.n def __getstate__(self): - return [ self.get( i ).soul for i in range(self.n) ] + raise NotImplementedError("A Combination's state is the set of rowids of the rows storing its constituants in the graphdb, which it doesn't know.") @classmethod def reconstitute(cls, state, graphdb): diff --git a/pygs/graphserver/graphdb.py b/pygs/graphserver/graphdb.py index 6b9c34f0..0f5e5587 100644 --- a/pygs/graphserver/graphdb.py +++ b/pygs/graphserver/graphdb.py @@ -36,10 +36,19 @@ def setup(self): def put_edge_payload(self, edgepayload, cc): if edgepayload.__class__ == Combination: + edge_state = [] + for component in edgepayload.components: - self.put_edge_payload( component, cc ) + rowid = self.put_edge_payload( component, cc ) + + edge_state.append( rowid ) + else: + edge_state = edgepayload.__getstate__() + - cc.execute( "INSERT INTO payloads (type, state) VALUES (?, ?)", ( cPickle.dumps( edgepayload.__class__ ), cPickle.dumps( edgepayload.__getstate__() ) ) ) + cc.execute( "INSERT INTO payloads (type, state) VALUES (?, ?)", + ( cPickle.dumps( edgepayload.__class__ ), + cPickle.dumps( edge_state ) ) ) return cc.lastrowid @@ -70,10 +79,12 @@ def populate(self, graph, reporter=None): c.execute( "INSERT INTO vertices VALUES (?)", (vv.label,) ) for ee in vv.outgoing: epid = self.put_edge_payload( ee.payload, c ) + c.execute( "INSERT INTO edges VALUES (?, ?, ?)", (ee.from_v.label, ee.to_v.label, epid) ) - + if hasattr(ee.payload, "__resources__"): for name, resource in ee.payload.__resources__(): + self.store( name, resource, c ) self.conn.commit()