Skip to content

Commit 94e49e5

Browse files
bogdandmbogdandm
committed
Global models module refactoring
1 parent 5f08308 commit 94e49e5

17 files changed

Lines changed: 574 additions & 473 deletions

‎json_to_models/cli.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
import json_to_models
1313
from json_to_models.dynamic_typing import ModelMeta, register_datetime_classes
1414
from json_to_models.generator import MetadataGenerator
15-
from json_to_models.models import ModelsStructureType, compose_models, compose_models_flat
15+
from json_to_models.models import ModelsStructureType
1616
from json_to_models.models.attr import AttrsModelCodeGenerator
1717
from json_to_models.models.base import GenericModelCodeGenerator, generate_code
1818
from json_to_models.models.dataclasses import DataclassModelCodeGenerator
19+
from json_to_models.models.structure import compose_models, compose_models_flat
1920
from json_to_models.registry import (
2021
ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry
2122
)

‎json_to_models/models/__init__.py‎

Lines changed: 6 additions & 220 deletions
Original file line numberDiff line numberDiff line change
@@ -1,229 +1,15 @@
1-
from collections import defaultdict
2-
from typing import Dict, Generic, Iterable, List, Set, Tuple, TypeVar, Union
1+
from enum import Enum
2+
from typing import Dict, List, Tuple
33

4-
from ..dynamic_typing import DOptional, ModelMeta, ModelPtr
4+
from ..dynamic_typing import ModelMeta
55

66
Index = str
77
ModelsStructureType = Tuple[List[dict], Dict[ModelMeta, ModelMeta]]
88

9-
T = TypeVar('T')
10-
11-
12-
class ListEx(list, Generic[T]):
13-
"""
14-
Extended list with shortcut methods
15-
"""
16-
17-
def safe_index(self, value: T):
18-
try:
19-
return self.index(value)
20-
except ValueError:
21-
return None
22-
23-
def _safe_indexes(self, *values: T):
24-
return [i for i in map(self.safe_index, values) if i is not None]
25-
26-
def insert_before(self, value: T, *before: T):
27-
ix = self._safe_indexes(*before)
28-
if not ix:
29-
raise ValueError
30-
pos = min(ix)
31-
self.insert(pos, value)
32-
return pos
33-
34-
def insert_after(self, value: T, *after: T):
35-
ix = self._safe_indexes(*after)
36-
if not ix:
37-
raise ValueError
38-
pos = max(ix) + 1
39-
self.insert(pos, value)
40-
return pos
41-
42-
43-
class PositionsDict(defaultdict):
44-
# Dict contains mapping Index -> position, where position is list index to insert nested element of Index
45-
INC = object()
46-
47-
def __init__(self, default_factory=int, **kwargs):
48-
super().__init__(default_factory, **kwargs)
49-
50-
def update_position(self, key: str, value: Union[object, int]):
51-
"""
52-
Shift all elements which are placed after updated one
53-
54-
:param key: Index or "root"
55-
:param value: Could be position or PositionsDict.INC to perform quick increment (x+=1)
56-
:return:
57-
"""
58-
if value is self.INC:
59-
value = self[key] + 1
60-
if key in self:
61-
old_value = self[key]
62-
delta = value - old_value
63-
else:
64-
old_value = value
65-
delta = 1
66-
for k, v in self.items():
67-
if k != key and v >= old_value:
68-
self[k] += delta
69-
self[key] = value
70-
71-
72-
def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType:
73-
"""
74-
Generate nested sorted models structure for internal usage.
75-
76-
:return: List of root models data, Map(child model -> root model) for absolute ref generation
77-
"""
78-
root_models = ListEx()
79-
root_nested_ix = 0
80-
structure_hash_table: Dict[Index, dict] = {
81-
key: {
82-
"model": model,
83-
"nested": ListEx(),
84-
"roots": list(extract_root(model)), # Indexes of root level models
85-
} for key, model in models_map.items()
86-
}
87-
# TODO: Test path_injections
88-
path_injections: Dict[ModelMeta, ModelMeta] = {}
89-
90-
for key, model in models_map.items():
91-
pointers = list(filter_pointers(model))
92-
has_root_pointers = len(pointers) != len(model.pointers)
93-
if not pointers:
94-
# Root level model
95-
if not has_root_pointers:
96-
raise Exception(f'Model {model.name} has no pointers')
97-
root_models.append(structure_hash_table[key])
98-
else:
99-
parents = {ptr.parent.index for ptr in pointers}
100-
struct = structure_hash_table[key]
101-
# Model is using by other models
102-
if has_root_pointers or len(parents) > 1 and len(struct["roots"]) > 1:
103-
# Model is using by different root models
104-
try:
105-
root_models.insert_before(
106-
struct,
107-
*(structure_hash_table[parent_key] for parent_key in struct["roots"])
108-
)
109-
except ValueError:
110-
root_models.insert(root_nested_ix, struct)
111-
root_nested_ix += 1
112-
elif len(parents) > 1 and len(struct["roots"]) == 1:
113-
# Model is using by single root model
114-
parent = structure_hash_table[struct["roots"][0]]
115-
parent["nested"].insert(0, struct)
116-
path_injections[struct["model"]] = parent["model"]
117-
else:
118-
# Model is using by only one model
119-
parent = structure_hash_table[next(iter(parents))]
120-
struct = structure_hash_table[key]
121-
parent["nested"].append(struct)
122-
123-
return root_models, path_injections
124-
125-
126-
def compose_models_flat(models_map: Dict[Index, ModelMeta]) -> ModelsStructureType:
127-
"""
128-
Generate flat sorted (by nesting level, ASC) models structure for internal usage.
129-
130-
:param models_map: Mapping (model index -> model meta instance).
131-
:return: List of root models data, Map(child model -> root model) for absolute ref generation
132-
"""
133-
root_models = ListEx()
134-
positions: PositionsDict[Index, int] = PositionsDict()
135-
top_level_models: Set[Index] = set()
136-
structure_hash_table: Dict[Index, dict] = {
137-
key: {
138-
"model": model,
139-
"nested": ListEx(),
140-
"roots": list(extract_root(model)), # Indexes of root level models
141-
} for key, model in models_map.items()
142-
}
143-
144-
for key, model in models_map.items():
145-
pointers = list(filter_pointers(model))
146-
has_root_pointers = len(pointers) != len(model.pointers)
147-
if not pointers:
148-
# Root level model
149-
if not has_root_pointers:
150-
raise Exception(f'Model {model.name} has no pointers')
151-
root_models.insert(positions["root"], structure_hash_table[key])
152-
top_level_models.add(key)
153-
positions.update_position("root", PositionsDict.INC)
154-
else:
155-
parents = {ptr.parent.index for ptr in pointers}
156-
struct = structure_hash_table[key]
157-
# Model is using by other models
158-
if has_root_pointers or len(parents) > 1 and len(struct["roots"]) >= 1:
159-
# Model is using by different root models
160-
if parents & top_level_models:
161-
parents.add("root")
162-
parents_positions = {positions[parent_key] for parent_key in parents
163-
if parent_key in positions}
164-
parents_joined = "#".join(sorted(parents))
165-
if parents_joined in positions:
166-
parents_positions.add(positions[parents_joined])
167-
pos = max(parents_positions) if parents_positions else len(root_models)
168-
positions.update_position(parents_joined, pos + 1)
169-
else:
170-
# Model is using by only one model
171-
parent = next(iter(parents))
172-
pos = positions.get(parent, len(root_models))
173-
positions.update_position(parent, pos + 1)
174-
positions.update_position(key, pos + 1)
175-
root_models.insert(pos, struct)
176-
177-
return root_models, {}
178-
179-
180-
def filter_pointers(model: ModelMeta) -> Iterable[ModelPtr]:
181-
"""
182-
Return iterator over pointers with not None parent
183-
"""
184-
return (ptr for ptr in model.pointers if ptr.parent)
185-
186-
187-
def extract_root(model: ModelMeta) -> Set[Index]:
188-
"""
189-
Return set of indexes of root models that are use given ``model`` directly or through another nested model.
190-
"""
191-
seen: Set[Index] = set()
192-
nodes: List[ModelPtr] = list(filter_pointers(model))
193-
roots: Set[Index] = set()
194-
while nodes:
195-
node = nodes.pop()
196-
seen.add(node.type.index)
197-
filtered = list(filter_pointers(node.parent))
198-
nodes.extend(ptr for ptr in filtered if ptr.type.index not in seen)
199-
if not filtered:
200-
roots.add(node.parent.index)
201-
return roots
202-
203-
204-
def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]:
205-
"""
206-
Split fields into required and optional groups
207-
208-
:return: two list of fields names: required fields, optional fields
209-
"""
210-
fields = model_meta.type
211-
required = []
212-
optional = []
213-
for key, meta in fields.items():
214-
if isinstance(meta, DOptional):
215-
optional.append(key)
216-
else:
217-
required.append(key)
218-
return required, optional
219-
220-
2219
INDENT = " " * 4
22210
OBJECTS_DELIMITER = "\n" * 3 # 2 blank lines
22311

22412

225-
def indent(string: str, lvl: int = 1, indent: str = INDENT) -> str:
226-
"""
227-
Indent all lines of string by ``indent * lvl``
228-
"""
229-
return "\n".join(indent * lvl + line for line in string.split("\n"))
13+
class ClassType(Enum):
14+
Dataclass = "dataclass"
15+
Attrs = "attrs"

‎json_to_models/models/base.py‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
from jinja2 import Template
77
from unidecode import unidecode
88

9-
from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields
10-
from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing
9+
from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER
10+
from .string_converters import get_string_field_paths
11+
from .structure import sort_fields
12+
from .utils import indent
13+
from ..dynamic_typing import (AbsoluteModelRef, ImportPathList, MetaData,
14+
ModelMeta, compile_imports, metadata_to_typing)
1115
from ..utils import cached_classmethod
1216

1317
METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD"
@@ -63,10 +67,24 @@ class {{ name }}:
6367
{%- endif -%}
6468
""")
6569

70+
STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})"
71+
% KWAGRS_TEMPLATE)
6672
FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}")
6773

68-
def __init__(self, model: ModelMeta, **kwargs):
74+
def __init__(self, model: ModelMeta, post_init_converters=False, **kwargs):
6975
self.model = model
76+
self.post_init_converters = post_init_converters
77+
78+
@cached_classmethod
79+
def convert_field_name(cls, name):
80+
if name in keywords_set:
81+
name += "_"
82+
name = unidecode(name)
83+
name = re.sub(r"\W", "", name)
84+
if not ('a' <= name[0].lower() <= 'z'):
85+
if '0' <= name[0] <= '9':
86+
name = ones[int(name[0])] + "_" + name[1:]
87+
return inflection.underscore(name)
7088

7189
def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[ImportPathList, str]:
7290
"""
@@ -90,18 +108,17 @@ def decorators(self) -> Tuple[ImportPathList, List[str]]:
90108
"""
91109
:return: List of imports and List of decorators code (without @)
92110
"""
93-
return [], []
94-
95-
@cached_classmethod
96-
def convert_field_name(cls, name):
97-
if name in keywords_set:
98-
name += "_"
99-
name = unidecode(name)
100-
name = re.sub(r"\W", "", name)
101-
if not ('a' <= name[0].lower() <= 'z'):
102-
if '0' <= name[0] <= '9':
103-
name = ones[int(name[0])] + "_" + name[1:]
104-
return inflection.underscore(name)
111+
imports, decorators = [], []
112+
if self.post_init_converters:
113+
str_fields = self.string_field_paths
114+
decorator_imports, decorator_kwargs = self.convert_strings_kwargs
115+
if str_fields and decorator_kwargs:
116+
imports.extend([
117+
*decorator_imports,
118+
('json_to_models.models.string_converters', ['convert_strings']),
119+
])
120+
decorators.append(self.STR_CONVERT_DECORATOR.render(str_fields=str_fields, kwargs=decorator_kwargs))
121+
return imports, decorators
105122

106123
def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]:
107124
"""
@@ -137,6 +154,23 @@ def fields(self) -> Tuple[ImportPathList, List[str]]:
137154
strings.append(self.FIELD.render(**data))
138155
return imports, strings
139156

157+
@property
158+
def string_field_paths(self) -> List[str]:
159+
"""
160+
Get paths for convert_strings function
161+
"""
162+
return [self.convert_field_name(name) + ('#' + '.'.join(path) if path else '')
163+
for name, path in get_string_field_paths(self.model)]
164+
165+
@property
166+
def convert_strings_kwargs(self) -> Tuple[ImportPathList, dict]:
167+
"""
168+
Override it to enable generation of string types converters
169+
170+
:return: Imports and Dict with kw-arguments for `json_to_models.models.string_converters.convert_strings` decorator.
171+
"""
172+
return [], {}
173+
140174

141175
def _generate_code(
142176
structure: List[dict],

0 commit comments

Comments
 (0)