Skip to content
This repository was archived by the owner on Mar 17, 2020. It is now read-only.

Commit c48d2e7

Browse files
ryanmatsJon Wayne Parrott
ryanmats
authored and
Jon Wayne Parrott
committed
Add datastore update schema sample (GoogleCloudPlatform#501)
1 parent 5d44e21 commit c48d2e7

6 files changed

Lines changed: 300 additions & 0 deletions

File tree

‎appengine/standard/ndb/schema_update/app.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
runtime: python27
2+
api_version: 1
3+
threadsafe: true
4+
5+
builtins:
6+
# Deferred is required to use google.appengine.ext.deferred.
7+
- deferred: on
8+
9+
handlers:
10+
- url: /.*
11+
script: main.app
12+
13+
libraries:
14+
- name: webapp2
15+
version: "2.5.2"
16+
- name: jinja2
17+
version: "2.6"

‎appengine/standard/ndb/schema_update/main.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Copyright 2016 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Sample application that shows how to perform a "schema migration" using
16+
Google Cloud Datastore.
17+
18+
This application uses one model named "Pictures" but two different versions
19+
of it. v2 contains two extra fields. The application shows how to
20+
populate these new fields onto entities that existed prior to adding the
21+
new fields to the model class.
22+
"""
23+
24+
import logging
25+
import os
26+
27+
from google.appengine.ext import deferred
28+
from google.appengine.ext import ndb
29+
import jinja2
30+
import webapp2
31+
32+
import models_v1
33+
import models_v2
34+
35+
36+
JINJA_ENVIRONMENT = jinja2.Environment(
37+
loader=jinja2.FileSystemLoader(
38+
os.path.join(os.path.dirname(__file__), 'templates')),
39+
extensions=['jinja2.ext.autoescape'],
40+
autoescape=True)
41+
42+
43+
class DisplayEntitiesHandler(webapp2.RequestHandler):
44+
"""Displays the current set of entities and options to add entities
45+
or update the schema."""
46+
def get(self):
47+
# Force ndb to use v2 of the model by re-loading it.
48+
reload(models_v2)
49+
50+
entities = models_v2.Picture.query().fetch()
51+
template_values = {
52+
'entities': entities,
53+
}
54+
55+
template = JINJA_ENVIRONMENT.get_template('index.html')
56+
self.response.write(template.render(template_values))
57+
58+
59+
class AddEntitiesHandler(webapp2.RequestHandler):
60+
"""Adds new entities using the v1 schema."""
61+
def post(self):
62+
# Force ndb to use v1 of the model by re-loading it.
63+
reload(models_v1)
64+
65+
# Save some example data.
66+
ndb.put_multi([
67+
models_v1.Picture(author='Alice', name='Sunset'),
68+
models_v1.Picture(author='Bob', name='Sunrise')
69+
])
70+
71+
self.response.write("""
72+
Entities created. <a href="/">View entities</a>.
73+
""")
74+
75+
76+
class UpdateSchemaHandler(webapp2.RequestHandler):
77+
"""Queues a task to start updating the model schema."""
78+
def post(self):
79+
deferred.defer(update_schema_task)
80+
self.response.write("""
81+
Schema update started. Check the console for task progress.
82+
<a href="/">View entities</a>.
83+
""")
84+
85+
86+
def update_schema_task(cursor=None, num_updated=0, batch_size=100):
87+
"""Task that handles updating the models' schema.
88+
89+
This is started by
90+
UpdateSchemaHandler. It scans every entity in the datastore for the
91+
Picture model and re-saves it so that it has the new schema fields.
92+
"""
93+
94+
# Force ndb to use v2 of the model by re-loading it.
95+
reload(models_v2)
96+
97+
# Get all of the entities for this Model.
98+
query = models_v2.Picture.query()
99+
pictures, cursor, more = query.fetch_page(batch_size, start_cursor=cursor)
100+
101+
to_put = []
102+
for picture in pictures:
103+
# Give the new fields default values.
104+
# If you added new fields and were okay with the default values, you
105+
# would not need to do this.
106+
picture.num_votes = 1
107+
picture.avg_rating = 5
108+
to_put.append(picture)
109+
110+
# Save the updated entities.
111+
if to_put:
112+
ndb.put_multi(to_put)
113+
num_updated += len(to_put)
114+
logging.info(
115+
'Put {} entities to Datastore for a total of {}'.format(
116+
len(to_put), num_updated))
117+
118+
# If there are more entities, re-queue this task for the next page.
119+
if more:
120+
deferred.defer(
121+
update_schema_task, cursor=query.cursor(), num_updated=num_updated)
122+
else:
123+
logging.debug(
124+
'update_schema_task complete with {0} updates!'.format(
125+
num_updated))
126+
127+
128+
app = webapp2.WSGIApplication([
129+
('/', DisplayEntitiesHandler),
130+
('/add_entities', AddEntitiesHandler),
131+
('/update_schema', UpdateSchemaHandler)])

‎appengine/standard/ndb/schema_update/main_test.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright 2015 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from google.appengine.ext import deferred
16+
import pytest
17+
import webtest
18+
19+
import main
20+
import models_v1
21+
import models_v2
22+
23+
24+
@pytest.fixture
25+
def app(testbed):
26+
yield webtest.TestApp(main.app)
27+
28+
29+
def test_app(app):
30+
response = app.get('/')
31+
assert response.status_int == 200
32+
33+
34+
def test_add_entities(app):
35+
response = app.post('/add_entities')
36+
assert response.status_int == 200
37+
response = app.get('/')
38+
assert response.status_int == 200
39+
assert 'Author: Bob' in response.body
40+
assert 'Name: Sunrise' in response.body
41+
assert 'Author: Alice' in response.body
42+
assert 'Name: Sunset' in response.body
43+
44+
45+
def test_update_schema(app, testbed):
46+
reload(models_v1)
47+
test_model = models_v1.Picture(author='Test', name='Test')
48+
test_model.put()
49+
50+
response = app.post('/update_schema')
51+
assert response.status_int == 200
52+
53+
# Run the queued task.
54+
tasks = testbed.taskqueue_stub.get_filtered_tasks()
55+
assert len(tasks) == 1
56+
deferred.run(tasks[0].payload)
57+
58+
# Check the updated items
59+
reload(models_v2)
60+
updated_model = test_model.key.get()
61+
assert updated_model.num_votes == 1
62+
assert updated_model.avg_rating == 5.0

‎appengine/standard/ndb/schema_update/models_v1.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2015 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from google.appengine.ext import ndb
16+
17+
18+
class Picture(ndb.Model):
19+
author = ndb.StringProperty()
20+
name = ndb.StringProperty(default='')

‎appengine/standard/ndb/schema_update/models_v2.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2015 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from google.appengine.ext import ndb
16+
17+
18+
class Picture(ndb.Model):
19+
author = ndb.StringProperty()
20+
name = ndb.StringProperty(default='')
21+
# Two new fields
22+
num_votes = ndb.IntegerProperty(default=0)
23+
avg_rating = ndb.FloatProperty(default=0)

‎appengine/standard/ndb/schema_update/templates/index.html‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{#
2+
# Copyright 2015 Google Inc. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#}
16+
<!DOCTYPE html>
17+
<html>
18+
<head>
19+
</head>
20+
<body>
21+
<p>If you've just added or updated entities, you may need to refresh
22+
the page to see the changes due to
23+
<a href="https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/">eventual consitency.</a></p>
24+
{% for entity in entities %}
25+
<p>
26+
Author: {{entity.author}},
27+
Name: {{entity.name}},
28+
{% if 'num_votes' in entity._values %}
29+
Votes: {{entity.num_votes}},
30+
{% endif %}
31+
{% if 'avg_rating' in entity._values %}
32+
Average Rating: {{entity.avg_rating}}
33+
{% endif %}
34+
</p>
35+
{% endfor %}
36+
{% if entities|length == 0 %}
37+
<form action="/add_entities" method="post" method="post">
38+
<input type="submit" value="Add Entities">
39+
</form>
40+
{% endif %}
41+
{% if entities|length > 0 %}
42+
<form action="/update_schema" method="post" method="post">
43+
<input type="submit" value="Update Schema">
44+
</form>
45+
{% endif %}
46+
</body>
47+
</html>

0 commit comments

Comments
 (0)