Skip to content

Commit 757a31a

Browse files
ibuleribuler
committed
[Update] 优化asset api, 获取时增加prefetch
1 parent b0710c4 commit 757a31a

7 files changed

Lines changed: 118 additions & 84 deletions

File tree

‎apps/assets/api/asset.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet):
4040
permission_classes = (IsSuperUserOrAppUser,)
4141

4242
def get_queryset(self):
43-
queryset = super().get_queryset()
43+
queryset = super().get_queryset()\
44+
.prefetch_related('labels', 'nodes')\
45+
.select_related('admin_user')
4446
admin_user_id = self.request.query_params.get('admin_user_id')
4547
node_id = self.request.query_params.get("node_id")
4648
show_current_asset = self.request.query_params.get("show_current_asset")
@@ -66,7 +68,6 @@ def get_queryset(self):
6668
queryset = queryset.filter(
6769
nodes__key__regex='^{}(:[0-9]+)*$'.format(node.key),
6870
).distinct()
69-
7071
return queryset
7172

7273

‎apps/assets/models/asset.py‎

Lines changed: 70 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -59,42 +59,70 @@ class Asset(models.Model):
5959
('Other', 'Other'),
6060
)
6161
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
62-
ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True)
63-
hostname = models.CharField(max_length=128, unique=True, verbose_name=_('Hostname'))
62+
ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'),
63+
db_index=True)
64+
hostname = models.CharField(max_length=128, unique=True,
65+
verbose_name=_('Hostname'))
6466
port = models.IntegerField(default=22, verbose_name=_('Port'))
65-
platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES, default='Linux', verbose_name=_('Platform'))
66-
domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets', verbose_name=_("Domain"), on_delete=models.SET_NULL)
67-
nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes"))
67+
platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES,
68+
default='Linux', verbose_name=_('Platform'))
69+
domain = models.ForeignKey("assets.Domain", null=True, blank=True,
70+
related_name='assets', verbose_name=_("Domain"),
71+
on_delete=models.SET_NULL)
72+
nodes = models.ManyToManyField('assets.Node', default=default_node,
73+
related_name='assets',
74+
verbose_name=_("Nodes"))
6875
is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
6976

7077
# Auth
71-
admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, null=True, verbose_name=_("Admin user"))
78+
admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT,
79+
null=True, verbose_name=_("Admin user"))
7280

7381
# Some information
74-
public_ip = models.GenericIPAddressField(max_length=32, blank=True, null=True, verbose_name=_('Public IP'))
75-
number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number'))
82+
public_ip = models.GenericIPAddressField(max_length=32, blank=True,
83+
null=True,
84+
verbose_name=_('Public IP'))
85+
number = models.CharField(max_length=32, null=True, blank=True,
86+
verbose_name=_('Asset number'))
7687

7788
# Collect
78-
vendor = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Vendor'))
79-
model = models.CharField(max_length=54, null=True, blank=True, verbose_name=_('Model'))
80-
sn = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Serial number'))
81-
82-
cpu_model = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU model'))
89+
vendor = models.CharField(max_length=64, null=True, blank=True,
90+
verbose_name=_('Vendor'))
91+
model = models.CharField(max_length=54, null=True, blank=True,
92+
verbose_name=_('Model'))
93+
sn = models.CharField(max_length=128, null=True, blank=True,
94+
verbose_name=_('Serial number'))
95+
96+
cpu_model = models.CharField(max_length=64, null=True, blank=True,
97+
verbose_name=_('CPU model'))
8398
cpu_count = models.IntegerField(null=True, verbose_name=_('CPU count'))
8499
cpu_cores = models.IntegerField(null=True, verbose_name=_('CPU cores'))
85-
memory = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Memory'))
86-
disk_total = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk total'))
87-
disk_info = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk info'))
88-
89-
os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS'))
90-
os_version = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('OS version'))
91-
os_arch = models.CharField(max_length=16, blank=True, null=True, verbose_name=_('OS arch'))
92-
hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw'))
93-
94-
labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels"))
95-
created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by'))
96-
date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created'))
97-
comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment'))
100+
memory = models.CharField(max_length=64, null=True, blank=True,
101+
verbose_name=_('Memory'))
102+
disk_total = models.CharField(max_length=1024, null=True, blank=True,
103+
verbose_name=_('Disk total'))
104+
disk_info = models.CharField(max_length=1024, null=True, blank=True,
105+
verbose_name=_('Disk info'))
106+
107+
os = models.CharField(max_length=128, null=True, blank=True,
108+
verbose_name=_('OS'))
109+
os_version = models.CharField(max_length=16, null=True, blank=True,
110+
verbose_name=_('OS version'))
111+
os_arch = models.CharField(max_length=16, blank=True, null=True,
112+
verbose_name=_('OS arch'))
113+
hostname_raw = models.CharField(max_length=128, blank=True, null=True,
114+
verbose_name=_('Hostname raw'))
115+
116+
labels = models.ManyToManyField('assets.Label', blank=True,
117+
related_name='assets',
118+
verbose_name=_("Labels"))
119+
created_by = models.CharField(max_length=32, null=True, blank=True,
120+
verbose_name=_('Created by'))
121+
date_created = models.DateTimeField(auto_now_add=True, null=True,
122+
blank=True,
123+
verbose_name=_('Date created'))
124+
comment = models.TextField(max_length=128, default='', blank=True,
125+
verbose_name=_('Comment'))
98126

99127
objects = AssetManager()
100128

@@ -121,6 +149,22 @@ def get_nodes(self):
121149
nodes = self.nodes.all() or [Node.root()]
122150
return nodes
123151

152+
@property
153+
def nodes_cache_key(self):
154+
key = "NODES_OF_{}".format(str(self.id))
155+
return key
156+
157+
def get_nodes_or_cache(self):
158+
cached = cache.get(self.nodes_cache_key)
159+
if cached is not None:
160+
return cached
161+
nodes = list(self.get_nodes())
162+
cache.set(self.nodes_cache_key, nodes, 3600)
163+
return nodes
164+
165+
def expire_nodes_cache(self):
166+
cache.delete(self.nodes_cache_key)
167+
124168
@property
125169
def hardware_info(self):
126170
if self.cpu_count:

‎apps/assets/models/node.py‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
class Node(models.Model):
1414
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
1515
key = models.CharField(unique=True, max_length=64, verbose_name=_("Key")) # '1:1:1:1'
16-
# value = models.CharField(
17-
# max_length=128, unique=True, verbose_name=_("Value")
18-
# )
1916
value = models.CharField(max_length=128, verbose_name=_("Value"))
2017
child_mark = models.IntegerField(default=0)
2118
date_create = models.DateTimeField(auto_now_add=True)
@@ -31,10 +28,11 @@ def name(self):
3128

3229
@property
3330
def full_value(self):
34-
if self == self.__class__.root():
31+
ancestor = [a.value for a in self.ancestor]
32+
if self.is_root():
3533
return self.value
36-
else:
37-
return '{} / {}'.format(self.parent.full_value, self.value)
34+
ancestor.append(self.value)
35+
return ' / '.join(ancestor)
3836

3937
@property
4038
def level(self):
@@ -118,7 +116,6 @@ def is_root(self):
118116
def parent(self):
119117
if self.key == "0" or not self.key.startswith("0"):
120118
return self.__class__.root()
121-
122119
parent_key = ":".join(self.key.split(":")[:-1])
123120
try:
124121
parent = self.__class__.objects.get(key=parent_key)
@@ -132,16 +129,17 @@ def parent(self, parent):
132129

133130
@property
134131
def ancestor(self):
135-
_key = self.key.split(':')
136-
ancestor_keys = []
137-
138132
if self.is_root():
139-
return [self.__class__.root()]
140-
141-
for i in range(len(_key)-1):
142-
_key.pop()
143-
ancestor_keys.append(':'.join(_key))
144-
return self.__class__.objects.filter(key__in=ancestor_keys)
133+
ancestor = self.__class__.objects.filter(key='0')
134+
else:
135+
_key = self.key.split(':')
136+
ancestor_keys = []
137+
for i in range(len(_key)-1):
138+
_key.pop()
139+
ancestor_keys.append(':'.join(_key))
140+
ancestor = self.__class__.objects.filter(key__in=ancestor_keys)
141+
ancestor = list(ancestor)
142+
return ancestor
145143

146144
@property
147145
def ancestor_with_self(self):

‎apps/assets/serializers/asset.py‎

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,34 +12,11 @@
1212
]
1313

1414

15-
class NodeTMPSerializer(serializers.ModelSerializer):
16-
parent = serializers.SerializerMethodField()
17-
assets_amount = serializers.SerializerMethodField()
18-
19-
class Meta:
20-
model = Node
21-
fields = ['id', 'key', 'value', 'parent', 'assets_amount', 'is_node']
22-
list_serializer_class = BulkListSerializer
23-
24-
@staticmethod
25-
def get_parent(obj):
26-
return obj.parent.id
27-
28-
@staticmethod
29-
def get_assets_amount(obj):
30-
return obj.get_all_assets().count()
31-
32-
def get_fields(self):
33-
fields = super().get_fields()
34-
field = fields["key"]
35-
field.required = False
36-
return fields
37-
38-
3915
class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer):
4016
"""
4117
资产的数据结构
4218
"""
19+
nodes = serializers.SerializerMethodField()
4320

4421
class Meta:
4522
model = Asset
@@ -54,6 +31,10 @@ def get_field_names(self, declared_fields, info):
5431
])
5532
return fields
5633

34+
@staticmethod
35+
def get_nodes(obj):
36+
return [n.id for n in obj.get_nodes_or_cache()]
37+
5738

5839
class AssetGrantedSerializer(serializers.ModelSerializer):
5940
"""

‎apps/assets/signals_handler.py‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -63,22 +63,31 @@ def on_system_user_assets_change(sender, instance=None, **kwargs):
6363

6464
@receiver(m2m_changed, sender=Asset.nodes.through)
6565
def on_asset_node_changed(sender, instance=None, **kwargs):
66-
if isinstance(instance, Asset) and kwargs['action'] == 'post_add':
67-
logger.debug("Asset node change signal received")
68-
nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
69-
system_users_assets = defaultdict(set)
70-
system_users = SystemUser.objects.filter(nodes__in=nodes)
71-
for system_user in system_users:
72-
system_users_assets[system_user].update({instance})
73-
for system_user, assets in system_users_assets.items():
74-
system_user.assets.add(*tuple(assets))
66+
if isinstance(instance, Asset):
67+
instance.expire_nodes_cache()
68+
if kwargs['action'] == 'post_add':
69+
logger.debug("Asset node change signal received")
70+
nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
71+
system_users_assets = defaultdict(set)
72+
system_users = SystemUser.objects.filter(nodes__in=nodes)
73+
# 清理节点缓存
74+
for system_user in system_users:
75+
system_users_assets[system_user].update({instance})
76+
for system_user, assets in system_users_assets.items():
77+
system_user.assets.add(*tuple(assets))
7578

7679

7780
@receiver(m2m_changed, sender=Asset.nodes.through)
7881
def on_node_assets_changed(sender, instance=None, **kwargs):
79-
if isinstance(instance, Node) and kwargs['action'] == 'post_add':
80-
logger.debug("Node assets change signal received")
82+
if isinstance(instance, Node):
8183
assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
82-
system_users = SystemUser.objects.filter(nodes=instance)
83-
for system_user in system_users:
84-
system_user.assets.add(*tuple(assets))
84+
# 清理资产节点缓存
85+
for asset in assets:
86+
asset.expire_nodes_cache()
87+
88+
if kwargs['action'] == 'post_add':
89+
logger.debug("Node assets change signal received")
90+
# 重新关联系统用户和资产的关系
91+
system_users = SystemUser.objects.filter(nodes=instance)
92+
for system_user in system_users:
93+
system_user.assets.add(*tuple(assets))

‎apps/perms/utils.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
class Tree:
1717
def __init__(self):
18-
self.__all_nodes = list(Node.objects.all())
18+
self.__all_nodes = list(Node.objects.all().prefetch_related('assets'))
1919
self.__node_asset_map = defaultdict(set)
2020
self.nodes = defaultdict(dict)
2121
self.root = Node.root()
@@ -134,7 +134,7 @@ def get_user_group_nodes_with_assets(cls, group):
134134
_assets = cls.get_user_group_assets(group)
135135
tree = Tree()
136136
for asset, _system_users in _assets.items():
137-
_nodes = asset.get_nodes()
137+
_nodes = asset.get_nodes_or_cache()
138138
tree.add_nodes(_nodes)
139139
for node in _nodes:
140140
tree.nodes[node][asset].update(_system_users)

‎jms‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ def start_gunicorn():
123123
'gunicorn', 'jumpserver.wsgi',
124124
'-b', bind,
125125
'-w', str(WORKERS),
126+
'-k', 'eventlet',
126127
'--access-logformat', log_format,
127128
'-p', pid_file,
128129
]

0 commit comments

Comments
 (0)