Skip to content

Commit b5a2e99

Browse files
yadvryadvr
committed
CLOUDSTACK-1037: Fix cloudmonkey's caching, autocompletion and printing
Signed-off-by: Rohit Yadav <bhaisaab@apache.org>
1 parent 5476391 commit b5a2e99

5 files changed

Lines changed: 126 additions & 130 deletions

File tree

‎tools/cli/cloudmonkey/cachemaker.py‎

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
try:
2020
import json
2121
import os
22-
import re
22+
import types
2323

24-
from requester import monkeyrequest
24+
from config import cache_file
2525
except ImportError, e:
2626
import sys
2727
print "ImportError", e
@@ -35,52 +35,79 @@ def getvalue(dictionary, key):
3535
return None
3636

3737

38-
def csv_str_as_list(string):
38+
def splitcsvstring(string):
3939
if string is not None:
4040
return filter(lambda x: x.strip() != '', string.split(','))
4141
else:
4242
return []
4343

4444

45-
def cachegen_from_file(json_file):
45+
def splitverbsubject(string):
46+
idx = 0
47+
for char in string:
48+
if char.islower():
49+
idx += 1
50+
else:
51+
break
52+
return string[:idx].lower(), string[idx:].lower()
53+
54+
55+
def savecache(apicache, json_file):
56+
"""
57+
Saves apicache dictionary as json_file, returns dictionary as indented str
58+
"""
59+
apicachestr = json.dumps(apicache, indent=2)
60+
with open(json_file, 'w') as cache_file:
61+
cache_file.write(apicachestr)
62+
return apicachestr
63+
64+
65+
def loadcache(json_file):
66+
"""
67+
Loads json file as dictionary, feeds it to monkeycache and spits result
68+
"""
4669
f = open(json_file, 'r')
4770
data = f.read()
4871
f.close()
4972
try:
50-
apis = json.loads(data)
73+
apicache = json.loads(data)
5174
except ValueError, e:
52-
print "Error processing json in cachegen()", e
53-
return cachegen(apis)
75+
print "Error processing json:", json_file, e
76+
return {}
77+
return apicache
5478

5579

56-
def cachegen(apis):
57-
pattern = re.compile("[A-Z]")
80+
def monkeycache(apis):
81+
"""
82+
Feed this a dictionary of api bananas, it spits out processed cache
83+
"""
84+
if isinstance(type(apis), types.NoneType):
85+
return {}
5886
responsekey = filter(lambda x: 'response' in x, apis.keys())
5987

6088
if len(responsekey) == 0:
61-
print "[cachegen] Invalid dictionary, has no response"
89+
print "[monkeycache] Invalid dictionary, has no response"
6290
return None
6391
if len(responsekey) != 1:
64-
print "[cachegen] Multiple responsekeys, chosing first one"
92+
print "[monkeycache] Multiple responsekeys, chosing first one"
6593

6694
responsekey = responsekey[0]
6795
verbs = set()
6896
cache = {}
6997
cache['count'] = getvalue(apis[responsekey], 'count')
98+
cache['asyncapis'] = []
7099

71100
for api in getvalue(apis[responsekey], 'api'):
72101
name = getvalue(api, 'name')
73-
response = getvalue(api, 'response')
74-
75-
idx = pattern.search(name).start()
76-
verb = name[:idx]
77-
subject = name[idx:]
102+
verb, subject = splitverbsubject(name)
78103

79104
apidict = {}
80105
apidict['name'] = name
81106
apidict['description'] = getvalue(api, 'description')
82107
apidict['isasync'] = getvalue(api, 'isasync')
83-
apidict['related'] = csv_str_as_list(getvalue(api, 'related'))
108+
if apidict['isasync']:
109+
cache['asyncapis'].append(name)
110+
apidict['related'] = splitcsvstring(getvalue(api, 'related'))
84111

85112
required = []
86113
apiparams = []
@@ -91,15 +118,16 @@ def cachegen(apis):
91118
apiparam['required'] = (getvalue(param, 'required') is True)
92119
apiparam['length'] = int(getvalue(param, 'length'))
93120
apiparam['type'] = getvalue(param, 'type')
94-
apiparam['related'] = csv_str_as_list(getvalue(param, 'related'))
121+
apiparam['related'] = splitcsvstring(getvalue(param, 'related'))
95122
if apiparam['required']:
96123
required.append(apiparam['name'])
97124
apiparams.append(apiparam)
98125

99126
apidict['requiredparams'] = required
100127
apidict['params'] = apiparams
101-
apidict['response'] = getvalue(api, 'response')
102-
cache[verb] = {subject: apidict}
128+
if verb not in cache:
129+
cache[verb] = {}
130+
cache[verb][subject] = apidict
103131
verbs.add(verb)
104132

105133
cache['verbs'] = list(verbs)
@@ -108,15 +136,15 @@ def cachegen(apis):
108136

109137
def main(json_file):
110138
"""
111-
cachegen.py creates a precache datastore of all available apis of
139+
cachemaker.py creates a precache datastore of all available apis of
112140
CloudStack and dumps the precache dictionary in an
113141
importable python module. This way we cheat on the runtime overhead of
114142
completing commands and help docs. This reduces the overall search and
115143
cache_miss (computation) complexity from O(n) to O(1) for any valid cmd.
116144
"""
117145
f = open("precache.py", "w")
118146
f.write("""# -*- coding: utf-8 -*-
119-
# Auto-generated code by cachegen.py
147+
# Auto-generated code by cachemaker.py
120148
# Licensed to the Apache Software Foundation (ASF) under one
121149
# or more contributor license agreements. See the NOTICE file
122150
# distributed with this work for additional information
@@ -133,13 +161,13 @@ def main(json_file):
133161
# KIND, either express or implied. See the License for the
134162
# specific language governing permissions and limitations
135163
# under the License.""")
136-
f.write("\nprecache = %s" % cachegen_from_file(json_file))
164+
f.write("\napicache = %s" % loadcache(json_file))
137165
f.close()
138166

139167
if __name__ == "__main__":
140-
json_file = 'listapis.json'
141-
if os.path.exists(json_file):
142-
main(json_file)
168+
print "[cachemaker] Pre-caching using user's cloudmonkey cache", cache_file
169+
if os.path.exists(cache_file):
170+
main(cache_file)
143171
else:
144-
pass
145-
#print "[ERROR] cli:cachegen is unable to locate %s" % json_file
172+
print "[cachemaker] Unable to cache apis, file not found", cache_file
173+
print "[cachemaker] Run cloudmonkey sync to generate cache"

‎tools/cli/cloudmonkey/cloudmonkey.py‎

Lines changed: 64 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -24,28 +24,32 @@
2424
import logging
2525
import os
2626
import pdb
27-
import re
2827
import shlex
2928
import sys
29+
import types
3030

3131
from urllib2 import HTTPError, URLError
3232
from httplib import BadStatusLine
3333

34-
from config import __version__, config_file
35-
from config import precached_verbs, read_config, write_config
34+
from config import __version__, cache_file
35+
from config import read_config, write_config
36+
3637
from printer import monkeyprint
3738
from requester import monkeyrequest
39+
from cachemaker import loadcache, savecache, monkeycache
40+
from cachemaker import splitverbsubject
3841

3942
from prettytable import PrettyTable
40-
from marvin.cloudstackConnection import cloudConnection
41-
from marvin.cloudstackException import cloudstackAPIException
42-
from marvin.cloudstackAPI import *
43-
from marvin import cloudstackAPI
4443
except ImportError, e:
4544
print "Import error in %s : %s" % (__name__, e)
4645
import sys
4746
sys.exit()
4847

48+
try:
49+
from precache import apicache
50+
except ImportError:
51+
apicache = {}
52+
4953
# Fix autocompletion issue, can be put in .pythonstartup
5054
try:
5155
import readline
@@ -60,23 +64,21 @@
6064

6165
log_fmt = '%(asctime)s - %(filename)s:%(lineno)s - [%(levelname)s] %(message)s'
6266
logger = logging.getLogger(__name__)
63-
completions = cloudstackAPI.__all__
6467

6568

6669
class CloudMonkeyShell(cmd.Cmd, object):
6770
intro = ("☁ Apache CloudStack 🐵 cloudmonkey " + __version__ +
6871
". Type help or ? to list commands.\n")
6972
ruler = "="
70-
apicache = {}
71-
# datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}}
72-
cache_verbs = precached_verbs
73+
cache_file = cache_file
74+
## datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}}
75+
#cache_verbs = apicache
7376
config_options = []
7477

75-
def __init__(self, pname, verbs):
78+
def __init__(self, pname):
7679
self.program_name = pname
77-
self.verbs = verbs
78-
7980
self.config_options = read_config(self.get_attr, self.set_attr)
81+
self.loadcache()
8082
self.prompt = self.prompt.strip() + " " # Cosmetic fix for prompt
8183

8284
logging.basicConfig(filename=self.log_file,
@@ -111,8 +113,27 @@ def cmdloop(self, intro=None):
111113
except KeyboardInterrupt:
112114
print("^C")
113115

116+
def loadcache(self):
117+
if os.path.exists(self.cache_file):
118+
self.apicache = loadcache(self.cache_file)
119+
else:
120+
self.apicache = apicache
121+
self.verbs = apicache['verbs']
122+
114123
def monkeyprint(self, *args):
115-
monkeyprint((self.color == 'true'), *args)
124+
output = ""
125+
try:
126+
for arg in args:
127+
if isinstance(type(arg), types.NoneType):
128+
continue
129+
output += str(arg)
130+
except Exception, e:
131+
print e
132+
133+
if self.color == 'true':
134+
monkeyprint(output)
135+
else:
136+
print output
116137

117138
def print_result(self, result, result_filter=None):
118139
if result is None or len(result) == 0:
@@ -186,6 +207,9 @@ def default(self, args):
186207
if self.pipe_runner(args):
187208
return
188209

210+
apiname = args.partition(' ')[0]
211+
verb, subject = splitverbsubject(apiname)
212+
189213
lexp = shlex.shlex(args.strip())
190214
lexp.whitespace = " "
191215
lexp.whitespace_split = True
@@ -196,7 +220,6 @@ def default(self, args):
196220
if next_val is None:
197221
break
198222
args.append(next_val)
199-
api_name = args[0]
200223

201224
args_dict = dict(map(lambda x: [x.partition("=")[0],
202225
x.partition("=")[2]],
@@ -207,22 +230,15 @@ def default(self, args):
207230
map(lambda x: x.strip(),
208231
args_dict.pop('filter').split(',')))
209232

210-
for attribute in args_dict.keys():
211-
setattr(api_cmd, attribute, args_dict[attribute])
212-
213-
#command = api_cmd()
214-
#missing_args = filter(lambda x: x not in args_dict.keys(),
215-
# command.required)
216-
217-
#if len(missing_args) > 0:
218-
# self.monkeyprint("Missing arguments: ", ' '.join(missing_args))
219-
# return
233+
missing_args = filter(lambda x: x not in args_dict.keys(),
234+
self.apicache[verb][subject]['requiredparams'])
220235

221-
isAsync = False
222-
#if "isAsync" in dir(command):
223-
# isAsync = (command.isAsync == "true")
236+
if len(missing_args) > 0:
237+
self.monkeyprint("Missing arguments: ", ' '.join(missing_args))
238+
return
224239

225-
result = self.make_request(api_name, args_dict, isAsync)
240+
result = self.make_request(apiname, args_dict,
241+
apiname in self.apicache['asyncapis'])
226242
if result is None:
227243
return
228244
try:
@@ -248,27 +264,29 @@ def completedefault(self, text, line, begidx, endidx):
248264
search_string = ""
249265

250266
if separator != " ": # Complete verb subjects
251-
autocompletions = self.cache_verbs[verb].keys()
267+
autocompletions = self.apicache[verb].keys()
252268
search_string = subject
253269
else: # Complete subject params
254270
autocompletions = map(lambda x: x + "=",
255-
self.cache_verbs[verb][subject][1])
271+
map(lambda x: x['name'],
272+
self.apicache[verb][subject]['params']))
256273
search_string = text
257274

258275
if self.tabularize == "true" and subject != "":
259276
autocompletions.append("filter=")
260277
return [s for s in autocompletions if s.startswith(search_string)]
261278

279+
262280
def do_sync(self, args):
263281
"""
264282
Asks cloudmonkey to discovery and sync apis available on user specified
265283
CloudStack host server which has the API discovery plugin, on failure
266284
it rollbacks last datastore or api precached datastore.
267285
"""
268286
response = self.make_request("listApis")
269-
f = open('test.json', "w")
270-
f.write(json.dumps(response))
271-
f.close()
287+
self.apicache = monkeycache(response)
288+
savecache(self.apicache, self.cache_file)
289+
self.loadcache()
272290

273291
def do_api(self, args):
274292
"""
@@ -282,11 +300,6 @@ def do_api(self, args):
282300
else:
283301
self.monkeyprint("Please use a valid syntax")
284302

285-
def complete_api(self, text, line, begidx, endidx):
286-
mline = line.partition(" ")[2]
287-
offs = len(mline) - len(text)
288-
return [s[offs:] for s in completions if s.startswith(mline)]
289-
290303
def do_set(self, args):
291304
"""
292305
Set config for cloudmonkey. For example, options can be:
@@ -387,19 +400,22 @@ def do_EOF(self, args):
387400

388401

389402
def main():
390-
pattern = re.compile("[A-Z]")
391-
verbs = list(set([x[:pattern.search(x).start()] for x in completions
392-
if pattern.search(x) is not None]).difference(['cloudstack']))
403+
verbs = []
404+
if os.path.exists(cache_file):
405+
verbs = loadcache(cache_file)['verbs']
406+
elif 'verbs' in apicache:
407+
verbs = apicache['verbs']
408+
393409
for verb in verbs:
394410
def add_grammar(verb):
395411
def grammar_closure(self, args):
396412
if self.pipe_runner("%s %s" % (verb, args)):
397413
return
398414
try:
399415
args_partition = args.partition(" ")
400-
res = self.cache_verbs[verb][args_partition[0]]
401-
cmd = res[0]
402-
helpdoc = res[2]
416+
api = self.apicache[verb][args_partition[0]]
417+
cmd = api['name']
418+
helpdoc = api['description']
403419
args = args_partition[2]
404420
except KeyError, e:
405421
self.monkeyprint("Error: invalid %s api arg" % verb, e)
@@ -412,10 +428,10 @@ def grammar_closure(self, args):
412428

413429
grammar_handler = add_grammar(verb)
414430
grammar_handler.__doc__ = "%ss resources" % verb.capitalize()
415-
grammar_handler.__name__ = 'do_' + verb
431+
grammar_handler.__name__ = 'do_' + str(verb)
416432
setattr(CloudMonkeyShell, grammar_handler.__name__, grammar_handler)
417433

418-
shell = CloudMonkeyShell(sys.argv[0], verbs)
434+
shell = CloudMonkeyShell(sys.argv[0])
419435
if len(sys.argv) > 1:
420436
shell.onecmd(' '.join(sys.argv[1:]))
421437
else:

0 commit comments

Comments
 (0)