#!/usr/bin/env python2

#

# Generate initialization data for built-in strings and objects.

#

# Supports two different initialization approaches:

#

# 1. Bit-packed format for unpacking strings and objects during

# heap or thread init into RAM-based structures. This is the

# default behavior.

#

# 2. Embedding strings and/or objects into a read-only data section

# at compile time. This is useful for low memory targets to reduce

# memory usage. Objects in data section will be immutable.

#

# Both of these have practical complications like endianness differences,

# pointer compression variants, object property table layout variants,

# and so on. Multiple #if defined()'d initializer sections are emitted

# to cover all supported alternatives.

#

import logging

import sys

logging.basicConfig(level=logging.INFO, stream=sys.stdout, format='%(name)-21s %(levelname)-7s %(message)s')

logger = logging.getLogger('genbuiltins.py')

logger.setLevel(logging.INFO)

import os

import re

import traceback

import json

import yaml

import math

import struct

import optparse

import copy

import logging

import dukutil

# Fixed seed for ROM strings, must match src-input/duk_heap_alloc.c.

DUK__FIXED_HASH_SEED = 0xabcd1234

# Base value for compressed ROM pointers, used range is [ROMPTR_FIRST,0xffff].

# Must match DUK_USE_ROM_PTRCOMP_FIRST (generated header checks).

ROMPTR_FIRST = 0xf800 # 2048 should be enough; now around ~1000 used

# ROM string table size

ROMSTR_LOOKUP_SIZE = 256

#

# Miscellaneous helpers

#

# Convert Unicode to bytes, identifying Unicode U+0000 to U+00FF as bytes.

# This representation is used in YAML metadata and allows invalid UTF-8 to

# be represented exactly (which is necessary).

def unicode_to_bytes(x):

if isinstance(x, str):

return x

tmp = ''

for c in x:

if ord(c) > 0xff:

raise Exception('invalid codepoint: %r' % x)

tmp += chr(ord(c))

assert(isinstance(tmp, str))

return tmp

# Convert bytes to Unicode, identifying bytes as U+0000 to U+00FF.

def bytes_to_unicode(x):

if isinstance(x, unicode):

return x

tmp = u''

for c in x:

tmp += unichr(ord(c))

assert(isinstance(tmp, unicode))

return tmp

# Convert all strings in an object to bytes recursively. Useful for

# normalizing all strings in a YAML document.

def recursive_strings_to_bytes(doc):

def f(x):

if isinstance(x, unicode):

return unicode_to_bytes(x)

if isinstance(x, dict):

res = {}

for k in x.keys():

res[f(k)] = f(x[k])

return res

if isinstance(x, list):

res = []

for e in x:

res.append(f(e))

return res

return x

return f(doc)

# Convert all strings in an object to from bytes to Unicode recursively.

# Useful for writing back JSON/YAML dumps.

def recursive_bytes_to_strings(doc):

def f(x):

if isinstance(x, str):

return bytes_to_unicode(x)

if isinstance(x, dict):

res = {}

for k in x.keys():

res[f(k)] = f(x[k])

return res

if isinstance(x, list):

res = []

for e in x:

res.append(f(e))

return res

return x

return f(doc)

# Check if string is an "array index" in ECMAScript terms.

def string_is_arridx(v):

is_arridx = False

try:

ival = int(v)

if ival >= 0 and ival <= 0xfffffffe and ('%d' % ival == v):

is_arridx = True

except ValueError:

pass

return is_arridx

#

# Metadata loading, merging, and other preprocessing

#

# Final metadata object contains merged and normalized objects and strings.

# Keys added include (see more below):

#

# strings_stridx: string objects which have a stridx, matches stridx index order

# objects_bidx: objects which have a bidx, matches bidx index order

# objects_ram_toplevel: objects which are top level for RAM init

#

# Various helper keys are also added, containing auxiliary object/string

# lists, lookup maps, etc. See code below for details of these.

#

def metadata_lookup_object(meta, obj_id):

return meta['_objid_to_object'][obj_id]

def metadata_lookup_object_and_index(meta, obj_id):

for i,t in enumerate(meta['objects']):

if t['id'] == obj_id:

return t, i

return None, None

def metadata_lookup_property(obj, key):

for p in obj['properties']:

if p['key'] == key:

return p

return None

def metadata_lookup_property_and_index(obj, key):

for i,t in enumerate(obj['properties']):

if t['key'] == key:

return t, i

return None, None

# Remove disabled objects and properties.

def metadata_remove_disabled(meta, active_opts):

objlist = []

count_disabled_object = 0

count_notneeded_object = 0

count_disabled_property = 0

count_notneeded_property = 0

def present_if_check(v):

pi = v.get('present_if', None)

if pi is None:

return True

if isinstance(pi, (str, unicode)):

pi = [ pi ]

if not isinstance(pi, list):

raise Exception('invalid present_if syntax: %r' % pi)

# Present if all listed options are true or unknown.

# Absent if any option is known to be false.

for opt in pi:

if active_opts.get(opt, None) == False:

return False

return True

for o in meta['objects']:

if o.get('disable', False):

logger.debug('Remove disabled object: %s' % o['id'])

count_disabled_object += 1

elif not present_if_check(o):

logger.debug('Removed object not needed in active configuration: %s' % o['id'])

count_notneeded_object += 1

else:

objlist.append(o)

props = []

for p in o['properties']:

if p.get('disable', False):

logger.debug('Remove disabled property: %s, object: %s' % (p['key'], o['id']))

count_disabled_property += 1

elif not present_if_check(p):

logger.debug('Removed property not needed in active configuration: %s, object: %s' % (p['key'], o['id']))

count_notneeded_property += 1

else:

props.append(p)

o['properties'] = props

meta['objects'] = objlist

if count_disabled_object + count_notneeded_object + count_disabled_property + count_notneeded_property > 0:

logger.info('Removed %d objects (%d disabled, %d not needed by config), %d properties (%d disabled, %d not needed by config)' % (count_disabled_object + count_notneeded_object, count_disabled_object, count_notneeded_object, count_disabled_property + count_notneeded_property, count_disabled_property, count_notneeded_property))

# Delete dangling references to removed/missing objects.

def metadata_delete_dangling_references_to_object(meta, obj_id):

for o in meta['objects']:

new_p = []

for p in o['properties']:

v = p['value']

ptype = None

if isinstance(v, dict):

ptype = p['value']['type']

delprop = False

if ptype == 'object' and v['id'] == obj_id:

delprop = True

if ptype == 'accessor' and v.get('getter_id') == obj_id:

p['getter_id'] = None

if ptype == 'accessor' and v.get('setter_id') == obj_id:

p['setter_id'] = None

# XXX: Should empty accessor (= no getter, no setter) be deleted?

# If so, beware of shorthand.

if delprop:

logger.debug('Deleted property %s of object %s, points to deleted object %s' % \

(p['key'], o['id'], obj_id))

else:

new_p.append(p)

o['properties'] = new_p

# Merge a user YAML file into current metadata.

def metadata_merge_user_objects(meta, user_meta):

if user_meta.has_key('add_objects'):

raise Exception('"add_objects" removed, use "objects" with "add: True"')

if user_meta.has_key('replace_objects'):

raise Exception('"replace_objects" removed, use "objects" with "replace: True"')

if user_meta.has_key('modify_objects'):

raise Exception('"modify_objects" removed, use "objects" with "modify: True"')

for o in user_meta.get('objects', []):

if o.get('disable', False):

logger.debug('Skip disabled object: %s' % o['id'])

continue

targ, targ_idx = metadata_lookup_object_and_index(meta, o['id'])

if o.get('delete', False):

logger.debug('Delete object: %s' % targ['id'])

if targ is None:

raise Exception('Cannot delete object %s which doesn\'t exist' % o['id'])

meta['objects'].pop(targ_idx)

metadata_delete_dangling_references_to_object(meta, targ['id'])

continue

if o.get('replace', False):

logger.debug('Replace object %s' % o['id'])

if targ is None:

logger.warning('object to be replaced doesn\'t exist, append new object')

meta['objects'].append(o)

else:

meta['objects'][targ_idx] = o

continue

if o.get('add', False) or not o.get('modify', False): # 'add' is the default

logger.debug('Add object %s' % o['id'])

if targ is not None:

raise Exception('Cannot add object %s which already exists' % o['id'])

meta['objects'].append(o)

continue

assert(o.get('modify', False)) # modify handling

if targ is None:

raise Exception('Cannot modify object %s which doesn\'t exist' % o['id'])

for k in sorted(o.keys()):

# Merge top level keys by copying over, except 'properties'

if k == 'properties':

continue

targ[k] = o[k]

for p in o.get('properties', []):

if p.get('disable', False):

logger.debug('Skip disabled property: %s' % p['key'])

continue

prop = None

prop_idx = None

prop, prop_idx = metadata_lookup_property_and_index(targ, p['key'])

if prop is not None:

if p.get('delete', False):

logger.debug('Delete property %s of %s' % (p['key'], o['id']))

targ['properties'].pop(prop_idx)

else:

logger.debug('Replace property %s of %s' % (p['key'], o['id']))

targ['properties'][prop_idx] = p

else:

if p.get('delete', False):

logger.debug('Deleting property %s of %s: doesn\'t exist, nop' % (p['key'], o['id']))

else:

logger.debug('Add property %s of %s' % (p['key'], o['id']))

targ['properties'].append(p)

# Replace 'symbol' keys and values with encoded strings.

def format_symbol(sym):

#print(repr(sym))

assert(isinstance(sym, dict))

assert(sym.get('type', None) == 'symbol')

variant = sym['variant']

if variant == 'global':

return '\x80' + sym['string']

elif variant == 'wellknown':

# Well known symbols use an empty suffix which never occurs for

# runtime local symbols.

return '\x81' + sym['string'] + '\xff'

elif variant == 'userhidden':

return '\xff' + sym['string']

elif variant == 'hidden': # hidden == Duktape hidden Symbol

return '\x82' + sym['string']

raise Exception('invalid symbol variant %r' % variant)

def metadata_normalize_symbol_strings(meta):

for o in meta['strings']:

if isinstance(o['str'], dict) and o['str'].get('type') == 'symbol':

o['str'] = format_symbol(o['str'])

#print('normalized symbol as string list element: %r', o)

for o in meta['objects']:

for p in o['properties']:

if isinstance(p['key'], dict) and p['key'].get('type') == 'symbol':

p['key'] = format_symbol(p['key'])

#print('normalized symbol as property key: %r', p)

if isinstance(p['value'], dict) and p['value'].get('type') == 'symbol':

p['value'] = format_symbol(p['value'])

#print('normalized symbol as property value: %r', p)

# Normalize nargs for top level functions by defaulting 'nargs' from 'length'.

def metadata_normalize_nargs_length(meta):

# Default 'nargs' from 'length' for top level function objects.

for o in meta['objects']:

if o.has_key('nargs'):

continue

if not o.get('callable', False):

continue

for p in o['properties']:

if p['key'] != 'length':

continue

logger.debug('Default nargs for top level: %r' % p)

assert(isinstance(p['value'], int))

o['nargs'] = p['value']

break

assert(o.has_key('nargs'))

# Default 'nargs' from 'length' for function property shorthand.

for o in meta['objects']:

for p in o['properties']:

if not (isinstance(p['value'], dict) and p['value']['type'] == 'function'):

continue

pval = p['value']

if not pval.has_key('length'):

logger.debug('Default length for function shorthand: %r' % p)

pval['length'] = 0

if not pval.has_key('nargs'):

logger.debug('Default nargs for function shorthand: %r' % p)

pval['nargs'] = pval['length']

# Prepare a list of built-in objects which need a runtime 'bidx'.

def metadata_prepare_objects_bidx(meta):

objlist = meta['objects']

meta['objects'] = []

meta['objects_bidx'] = []

# Objects have a 'bidx: true' if they need a DUK_BIDX_xxx constant

# and need to be present in thr->builtins[]. The list is already

# stripped of built-in objects which are not needed based on config.

# Ideally we'd scan the actually needed indices from the source

# but since some usage is inside #if defined()s that's not trivial.

for obj in objlist:

if obj.get('bidx', False):

obj['bidx_used'] = True

meta['objects'].append(obj)

meta['objects_bidx'].append(obj)

# Append remaining objects.

for obj in objlist:

if obj.get('bidx_used', False):

# Already in meta['objects'].

pass

else:

meta['objects'].append(obj)

# Normalize metadata property shorthand. For example, if a property value

# is a shorthand function, create a function object and change the property

# to point to that function object.

def metadata_normalize_shorthand(meta):

# Gather objects through the top level built-ins list.

objs = []

subobjs = []

def getSubObject():

obj = {}

obj['id'] = 'subobj_%d' % len(subobjs) # synthetic ID

obj['properties'] = []

obj['auto_generated'] = True # mark as autogenerated (just FYI)

subobjs.append(obj)

return obj

def decodeFunctionShorthand(funprop):

# Convert the built-in function property "shorthand" into an actual

# object for ROM built-ins.

assert(funprop['value']['type'] == 'function')

val = funprop['value']

obj = getSubObject()

props = obj['properties']

obj['native'] = val['native']

obj['nargs'] = val.get('nargs', val['length'])

obj['varargs'] = val.get('varargs', False)

obj['magic'] = val.get('magic', 0)

obj['internal_prototype'] = 'bi_function_prototype'

obj['class'] = 'Function'

obj['callable'] = val.get('callable', True)

obj['constructable'] = val.get('constructable', False)

obj['special_call'] = val.get('special_call', False)

fun_name = val.get('name', funprop['key'])

props.append({ 'key': 'length', 'value': val['length'], 'attributes': 'c' }) # Configurable in ES2015

props.append({ 'key': 'name', 'value': fun_name, 'attributes': 'c' }) # Configurable in ES2015

return obj

def addAccessor(funprop, magic, nargs, length, name, native_func):

assert(funprop['value']['type'] == 'accessor')

obj = getSubObject()

props = obj['properties']

obj['native'] = native_func

obj['nargs'] = nargs

obj['varargs'] = False

obj['magic'] = magic

obj['internal_prototype'] = 'bi_function_prototype'

obj['class'] = 'Function'

obj['callable'] = val.get('callable', True)

obj['constructable'] = val.get('constructable', False)

assert(obj.get('special_call', False) == False)

# Shorthand accessors are minimal and have no .length or .name

# right now. Use longhand if these matter.

#props.append({ 'key': 'length', 'value': length, 'attributes': 'c' })

#props.append({ 'key': 'name', 'value': name, 'attributes': 'c' })

return obj

def decodeGetterShorthand(key, funprop):

assert(funprop['value']['type'] == 'accessor')

val = funprop['value']

if not val.has_key('getter'):

return None

return addAccessor(funprop,

val['getter_magic'],

val['getter_nargs'],

val.get('getter_length', 0),

key,

val['getter'])

def decodeSetterShorthand(key, funprop):

assert(funprop['value']['type'] == 'accessor')

val = funprop['value']

if not val.has_key('setter'):

return None

return addAccessor(funprop,

val['setter_magic'],

val['setter_nargs'],

val.get('setter_length', 0),

key,

val['setter'])

def decodeStructuredValue(val):

logger.debug('Decode structured value: %r' % val)

if isinstance(val, (int, long, float, str)):

return val # as is

elif isinstance(val, (dict)):

# Object: decode recursively

obj = decodeStructuredObject(val)

return { 'type': 'object', 'id': obj['id'] }

elif isinstance(val, (list)):

raise Exception('structured shorthand does not yet support array literals')

else:

raise Exception('unsupported value in structured shorthand: %r' % v)

def decodeStructuredObject(val):

# XXX: We'd like to preserve dict order from YAML source but

# Python doesn't do that. Use sorted order to make the result

# deterministic. User can always use longhand for exact

# property control.

logger.debug('Decode structured object: %r' % val)

obj = getSubObject()

obj['class'] = 'Object'

obj['internal_prototype'] = 'bi_object_prototype'

props = obj['properties']

keys = sorted(val.keys())

for k in keys:

logger.debug('Decode property %s' % k)

prop = { 'key': k, 'value': decodeStructuredValue(val[k]), 'attributes': 'wec' }

props.append(prop)

return obj

def decodeStructuredShorthand(structprop):

assert(structprop['value']['type'] == 'structured')

val = structprop['value']['value']

return decodeStructuredValue(val)

def clonePropShared(prop):

res = {}

for k in [ 'key', 'attributes', 'auto_lightfunc' ]:

if prop.has_key(k):

res[k] = prop[k]

return res

for idx,obj in enumerate(meta['objects']):

props = []

repl_props = []

for val in obj['properties']:

# Date.prototype.toGMTString must point to the same Function object

# as Date.prototype.toUTCString, so special case hack it here.

if obj['id'] == 'bi_date_prototype' and val['key'] == 'toGMTString':

logger.debug('Skip Date.prototype.toGMTString')

continue

if isinstance(val['value'], dict) and val['value']['type'] == 'function':

# Function shorthand.

subfun = decodeFunctionShorthand(val)

prop = clonePropShared(val)

prop['value'] = { 'type': 'object', 'id': subfun['id'] }

repl_props.append(prop)

elif isinstance(val['value'], dict) and val['value']['type'] == 'accessor' and \

(val['value'].has_key('getter') or val['value'].has_key('setter')):

# Accessor normal and shorthand forms both use the type 'accessor',

# but are differentiated by properties.

sub_getter = decodeGetterShorthand(val['key'], val)

sub_setter = decodeSetterShorthand(val['key'], val)

prop = clonePropShared(val)

prop['value'] = { 'type': 'accessor' }

if sub_getter is not None:

prop['value']['getter_id'] = sub_getter['id']

if sub_setter is not None:

prop['value']['setter_id'] = sub_setter['id']

assert('a' in prop['attributes']) # If missing, weird things happen runtime

logger.debug('Expand accessor shorthand: %r -> %r' % (val, prop))

repl_props.append(prop)

elif isinstance(val['value'], dict) and val['value']['type'] == 'structured':

# Structured shorthand.

subval = decodeStructuredShorthand(val)

prop = clonePropShared(val)

prop['value'] = subval

repl_props.append(prop)

logger.debug('Decoded structured shorthand for object %s, property %s' % (obj['id'], val['key']))

elif isinstance(val['value'], dict) and val['value']['type'] == 'buffer':

# Duktape buffer type not yet supported.

raise Exception('Buffer type not yet supported for builtins: %r' % val)

elif isinstance(val['value'], dict) and val['value']['type'] == 'pointer':

# Duktape pointer type not yet supported.

raise Exception('Pointer type not yet supported for builtins: %r' % val)

else:

# Property already in normalized form.

repl_props.append(val)

if obj['id'] == 'bi_date_prototype' and val['key'] == 'toUTCString':

logger.debug('Clone Date.prototype.toUTCString to Date.prototype.toGMTString')

prop2 = copy.deepcopy(repl_props[-1])

prop2['key'] = 'toGMTString'

repl_props.append(prop2)

# Replace properties with a variant where function properties

# point to built-ins rather than using an inline syntax.

obj['properties'] = repl_props

len_before = len(meta['objects'])

meta['objects'] += subobjs

len_after = len(meta['objects'])

logger.debug('Normalized metadata shorthand, %d objects -> %d final objects' % (len_before, len_after))

# Normalize property attribute order, default attributes, etc.

def metadata_normalize_property_attributes(meta):

for o in meta['objects']:

for p in o['properties']:

orig_attrs = p.get('attributes', None)

is_accessor = (isinstance(p['value'], dict) and p['value']['type'] == 'accessor')

# If missing, set default attributes.

attrs = orig_attrs

if attrs is None:

if is_accessor:

attrs = 'ca' # accessor default is configurable

else:

attrs = 'wc' # default is writable, configurable

logger.debug('Defaulted attributes of %s/%s to %s' % (o['id'], p['key'], attrs))

# Decode flags to normalize their order in the end.

writable = 'w' in attrs

enumerable = 'e' in attrs

configurable = 'c' in attrs

accessor = 'a' in attrs

# Force 'accessor' attribute for accessors.

if is_accessor and not accessor:

logger.debug('Property %s is accessor but has no "a" attribute, add attribute' % p['key'])

accessor = True

# Normalize order and write back.

attrs = ''

if writable:

attrs += 'w'

if enumerable:

attrs += 'e'

if configurable:

attrs += 'c'

if accessor:

attrs += 'a'

p['attributes'] = attrs

if orig_attrs != attrs:

logger.debug('Updated attributes of %s/%s from %r to %r' % (o['id'], p['key'], orig_attrs, attrs))

pass

# Normalize ROM property attributes.

def metadata_normalize_rom_property_attributes(meta):

for o in meta['objects']:

for p in o['properties']:

# ROM properties must not be configurable (runtime code

# depends on this). Writability is kept so that instance

# objects can override parent properties.

p['attributes'] = p['attributes'].replace('c', '')

# Add a 'name' property for all top level functions; expected by RAM

# initialization code.

def metadata_normalize_ram_function_names(meta):

num_added = 0

for o in meta['objects']:

if not o.get('callable', False):

continue

name_prop = None

for p in o['properties']:

if p['key'] == 'name':

name_prop = p

break

if name_prop is None:

num_added += 1

logger.debug('Adding missing "name" property for function %s' % o['id'])

o['properties'].append({ 'key': 'name', 'value': '', 'attributes': 'c' })

if num_added > 0:

logger.debug('Added missing "name" property for %d functions' % num_added)

# Add a built-in objects list for RAM initialization.

def metadata_add_ram_filtered_object_list(meta):

# For RAM init data to support user objects, we need to prepare a

# filtered top level object list, containing only those objects which

# need a value stack index during duk_hthread_builtins.c init process.

#

# Objects in meta['objects'] which are covered by inline property

# notation in the init data (this includes e.g. member functions like

# Math.cos) must not be present.

objlist = []

for o in meta['objects']:

keep = o.get('bidx_used', False)

if o.has_key('native') and not o.has_key('bidx'):

# Handled inline by run-time init code

pass

else:

# Top level object

keep = True

if keep:

objlist.append(o)

logger.debug('Filtered RAM object list: %d objects with bidx, %d total top level objects' % \

(len(meta['objects_bidx']), len(objlist)))

meta['objects_ram_toplevel'] = objlist

# Add missing strings into strings metadata. For example, if an object

# property key is not part of the strings list, append it there. This

# is critical for ROM builtins because all property keys etc must also

# be in ROM.

def metadata_normalize_missing_strings(meta, user_meta):

# We just need plain strings here.

strs_have = {}

for s in meta['strings']:

strs_have[s['str']] = True

# For ROM builtins all the strings must be in the strings list,

# so scan objects for any strings not explicitly listed in metadata.

for idx, obj in enumerate(meta['objects']):

for prop in obj['properties']:

key = prop['key']

if not strs_have.get(key):

logger.debug('Add missing string: %r' % key)

meta['strings'].append({ 'str': key, '_auto_add_ref': True })

strs_have[key] = True

if prop.has_key('value') and isinstance(prop['value'], (str, unicode)):

val = unicode_to_bytes(prop['value']) # should already be, just in case

if not strs_have.get(val):

logger.debug('Add missing string: %r' % val)

meta['strings'].append({ 'str': val, '_auto_add_ref': True })

strs_have[val] = True

# Force user strings to be in ROM data.

for s in user_meta.get('add_forced_strings', []):

if not strs_have.get(s['str']):

logger.debug('Add user string: %r' % s['str'])

s['_auto_add_user'] = True

meta['strings'].append(s)

# Convert built-in function properties into lightfuncs where applicable.

def metadata_convert_lightfuncs(meta):

num_converted = 0

num_skipped = 0

for o in meta['objects']:

for p in o['properties']:

v = p['value']

ptype = None

if isinstance(v, dict):

ptype = p['value']['type']

if ptype != 'object':

continue

targ, targ_idx = metadata_lookup_object_and_index(meta, p['value']['id'])

reasons = []

if not targ.get('callable', False):

reasons.append('not-callable')

#if targ.get('constructable', False):

# reasons.append('constructable')

lf_len = 0

for p2 in targ['properties']:

# Don't convert if function has more properties than

# we're willing to sacrifice.

logger.debug(' - Check %r . %s' % (o.get('id', None), p2['key']))

if p2['key'] == 'length' and isinstance(p2['value'], (int, long)):

lf_len = p2['value']

if p2['key'] not in [ 'length', 'name' ]:

reasons.append('nonallowed-property')

if not p.get('auto_lightfunc', True):

logger.debug('Automatic lightfunc conversion rejected for key %s, explicitly requested in metadata' % p['key'])

reasons.append('no-auto-lightfunc')

# lf_len comes from actual property table (after normalization)

if targ.has_key('magic'):

try:

# Magic values which resolve to 'bidx' indices cannot

# be resolved here yet, because the bidx map is not

# yet ready. If so, reject the lightfunc conversion

# for now. In practice this doesn't matter.

lf_magic = resolve_magic(targ.get('magic'), {}) # empty map is a "fake" bidx map

logger.debug('resolved magic ok -> %r' % lf_magic)

except Exception, e:

logger.debug('Failed to resolve magic for %r: %r' % (p['key'], e))

reasons.append('magic-resolve-failed')

lf_magic = 0xffffffff # dummy, will be out of bounds

else:

lf_magic = 0

if targ.get('varargs', True):

lf_nargs = None

lf_varargs = True

else:

lf_nargs = targ['nargs']

lf_varargs = False

if lf_len < 0 or lf_len > 15:

logger.debug('lf_len out of bounds: %r' % lf_len)

reasons.append('len-bounds')

if lf_magic < -0x80 or lf_magic > 0x7f:

logger.debug('lf_magic out of bounds: %r' % lf_magic)

reasons.append('magic-bounds')

if not lf_varargs and (lf_nargs < 0 or lf_nargs > 14):

logger.debug('lf_nargs out of bounds: %r' % lf_nargs)

reasons.append('nargs-bounds')

if len(reasons) > 0:

logger.debug('Don\'t convert to lightfunc: %r %r (%r): %r' % (o.get('id', None), p.get('key', None), p['value']['id'], reasons))

num_skipped += 1

continue

p_id = p['value']['id']

p['value'] = {

'type': 'lightfunc',

'native': targ['native'],

'length': lf_len,

'magic': lf_magic,

'nargs': lf_nargs,

'varargs': lf_varargs

}

logger.debug(' - Convert to lightfunc: %r %r (%r) -> %r' % (o.get('id', None), p.get('key', None), p_id, p['value']))

num_converted += 1

logger.debug('Converted %d built-in function properties to lightfuncs, %d skipped as non-eligible' % (num_converted, num_skipped))

# Detect objects not reachable from any object with a 'bidx'. This is usually

# a user error because such objects can't be reached at runtime so they're

# useless in RAM or ROM init data.

def metadata_remove_orphan_objects(meta):

reachable = {}

for o in meta['objects']:

if o.get('bidx_used', False):

reachable[o['id']] = True

while True:

reachable_count = len(reachable.keys())

def _markId(obj_id):

if obj_id is None:

return

reachable[obj_id] = True

for o in meta['objects']:

if not reachable.has_key(o['id']):

continue

_markId(o.get('internal_prototype', None))

for p in o['properties']:

# Shorthand has been normalized so no need

# to support it here.

v = p['value']

ptype = None

if isinstance(v, dict):

ptype = p['value']['type']

if ptype == 'object':

_markId(v['id'])

if ptype == 'accessor':

_markId(v.get('getter_id'))

_markId(v.get('setter_id'))

logger.debug('Mark reachable: reachable count initially %d, now %d' % \

(reachable_count, len(reachable.keys())))

if reachable_count == len(reachable.keys()):

break

num_deleted = 0

deleted = True

while deleted:

deleted = False

for i,o in enumerate(meta['objects']):

if not reachable.has_key(o['id']):

logger.debug('object %s not reachable, dropping' % o['id'])

meta['objects'].pop(i)

deleted = True

num_deleted += 1

break

if num_deleted > 0:

logger.debug('Deleted %d unreachable objects' % num_deleted)

# Add C define names for builtin strings. These defines are added to all

# strings, even when they won't get a stridx because the define names are

# used to autodetect referenced strings.

def metadata_add_string_define_names(strlist, special_defs):

for s in strlist:

v = s['str']

if special_defs.has_key(v):

s['define'] = 'DUK_STRIDX_' + special_defs[v]

continue

if len(v) >= 1 and v[0] == '\x82':

pfx = 'DUK_STRIDX_INT_'

v = v[1:]

elif len(v) >= 1 and v[0] == '\x81' and v[-1] == '\xff':

pfx = 'DUK_STRIDX_WELLKNOWN_'

v = v[1:-1]

else:

pfx = 'DUK_STRIDX_'

t = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', v) # add underscores: aB -> a_B

t = re.sub(r'\.', '_', t) # replace . with _, e.g. Symbol.iterator

s['define'] = pfx + t.upper()

logger.debug('stridx define: ' + s['define'])

# Add a 'stridx_used' flag for strings which need a stridx.

def metadata_add_string_used_stridx(strlist, used_stridx_meta):

defs_needed = {}

defs_found = {}

for s in used_stridx_meta['used_stridx_defines']:

defs_needed[s] = True

# strings whose define is referenced

for s in strlist:

if s.has_key('define') and defs_needed.has_key(s['define']):

s['stridx_used'] = True

defs_found[s['define']] = True

# duk_lexer.h needs all reserved words

for s in strlist:

if s.get('reserved_word', False):

s['stridx_used'] = True

# ensure all needed defines are provided

defs_found['DUK_STRIDX_START_RESERVED'] = True # special defines provided automatically

defs_found['DUK_STRIDX_START_STRICT_RESERVED'] = True

defs_found['DUK_STRIDX_END_RESERVED'] = True

defs_found['DUK_STRIDX_TO_TOK'] = True

for k in sorted(defs_needed.keys()):

if not defs_found.has_key(k):

raise Exception('source code needs define %s not provided by strings' % repr(k))

# Merge duplicate strings in string metadata.

def metadata_merge_string_entries(strlist):

# The raw string list may contain duplicates so merge entries.

# The list is processed in reverse because the last entry should

# "win" and keep its place (this matters for reserved words).

strs = []

str_map = {} # plain string -> object in strs[]

tmp = copy.deepcopy(strlist)

tmp.reverse()

for s in tmp:

prev = str_map.get(s['str'])

if prev is not None:

for k in s.keys():

if prev.has_key(k) and prev[k] != s[k]:

raise Exception('fail to merge string entry, conflicting keys: %r <-> %r' % (prev, s))

prev[k] = s[k]

else:

strs.append(s)

str_map[s['str']] = s

strs.reverse()

return strs

# Order builtin strings (strings with a stridx) into an order satisfying

# multiple constraints.

def metadata_order_builtin_strings(input_strlist, keyword_list, strip_unused_stridx=False):

# Strings are ordered in the result as follows:

# 1. Non-reserved words requiring 8-bit indices

# 2. Non-reserved words not requiring 8-bit indices

# 3. Reserved words in non-strict mode only

# 4. Reserved words in strict mode

#

# Reserved words must follow an exact order because they are

# translated to/from token numbers by addition/subtraction.

# Some strings require an 8-bit index and must be in the

# beginning.

tmp_strs = []

for s in copy.deepcopy(input_strlist):

if not s.get('stridx_used', False):

# Drop strings which are not actually needed by src-input/*.(c|h).

# Such strings won't be in heap->strs[] or ROM legacy list.

pass

else:

tmp_strs.append(s)

# The reserved word list must match token order in duk_lexer.h

# exactly, so pluck them out first.

str_index = {}

kw_index = {}

keywords = []

strs = []

for idx,s in enumerate(tmp_strs):

str_index[s['str']] = s

for idx,s in enumerate(keyword_list):

keywords.append(str_index[s])

kw_index[s] = True

for idx,s in enumerate(tmp_strs):

if not kw_index.has_key(s['str']):

strs.append(s)

# Sort the strings by category number; within category keep

# previous order.

for idx,s in enumerate(strs):

s['_idx'] = idx # for ensuring stable sort

def req8Bit(s):

return s.get('class_name', False) # currently just class names

def getCat(s):

req8 = req8Bit(s)

if s.get('reserved_word', False):

# XXX: unused path now, because keywords are "plucked out"

# explicitly.

assert(not req8)

if s.get('future_reserved_word_strict', False):

return 4

else:

return 3

elif req8:

return 1

else:

return 2