Skip to content

Commit 201f9e1

Browse files
AyutoAyuto
authored
Add data updater (Source-Python-Dev-Team#212)
* Added core.updater module * Added data updater to setup routine Added setting to disable auto data updates. Added debug option to find out which data files are being accessed before the data has been updated. * Log file is now getting cleared * Changed setup routines to load data files later * Use our standard * Renamed updater to update * Fixed an error when fire_output is not found * Added logging to the update functions * Added reloading languages.ini * Removed unused import
1 parent 8e5ed1c commit 201f9e1

9 files changed

Lines changed: 272 additions & 114 deletions

File tree

‎addons/source-python/packages/source-python/__init__.py‎

Lines changed: 109 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,45 @@
2626
# all respects for all other code used. Additionally, the Source.Python
2727
# Development Team grants this exception to all derivative works.
2828

29+
30+
# =============================================================================
31+
# >> FILE ACCESS LOGGER
32+
# =============================================================================
33+
# If True, all calls to open() with a path to a Source.Python data file will be
34+
# logged in ../logs/file_access.log. The log entry will contain the file that
35+
# is being accessed and a full stack trace. The logger will be removed as soon
36+
# as setup_data_update() is called.
37+
# This is a debug option to ensure that no data files are being accessed before
38+
# the data has been updated. Release builds should have this option set to
39+
# False.
40+
LOG_FILE_OPERATIONS = False
41+
42+
if LOG_FILE_OPERATIONS:
43+
import builtins
44+
import traceback
45+
from paths import SP_DATA_PATH
46+
from paths import LOG_PATH
47+
48+
LOG_FILE = LOG_PATH / 'file_access.log'
49+
50+
# Clear log file
51+
LOG_FILE.open('w').close()
52+
53+
old_open = builtins.open
54+
55+
def new_open(f, *args, **kwargs):
56+
if isinstance(f, str) and f.startswith(SP_DATA_PATH):
57+
print(f)
58+
with LOG_FILE.open('a') as log_f:
59+
log_f.write('File access: {}\n'.format(f))
60+
traceback.print_stack(file=log_f)
61+
log_f.write('\n\n')
62+
63+
return old_open(f, *args, **kwargs)
64+
65+
builtins.open = new_open
66+
67+
2968
# =============================================================================
3069
# >> IMPORTS
3170
# =============================================================================
@@ -46,8 +85,10 @@ def load():
4685
setup_stdout_redirect()
4786
setup_core_settings()
4887
setup_logging()
49-
setup_hooks()
88+
setup_exception_hooks()
89+
setup_data_update()
5090
setup_translations()
91+
setup_data()
5192
setup_global_pointers()
5293
setup_sp_command()
5394
setup_auth()
@@ -65,6 +106,71 @@ def unload():
65106
unload_auth()
66107

67108

109+
# =============================================================================
110+
# >> DATA UPDATE
111+
# =============================================================================
112+
def setup_data_update():
113+
"""Setup data update."""
114+
_sp_logger.log_debug('Setting up data update...')
115+
116+
if LOG_FILE_OPERATIONS:
117+
builtins.open = old_open
118+
119+
from core.settings import _core_settings
120+
121+
if not _core_settings.auto_data_update:
122+
_sp_logger.log_debug('Automatic data updates are disable.')
123+
return
124+
125+
_sp_logger.log_info('Checking for data updates...')
126+
127+
from core.update import is_new_data_available, update_data
128+
from translations.manager import language_manager
129+
130+
try:
131+
if is_new_data_available():
132+
_sp_logger.log_info('New data is available. Downloading...')
133+
update_data()
134+
135+
# languages.ini is loaded before the data has been updated. Thus,
136+
# we need to reload the file.
137+
language_manager.reload()
138+
else:
139+
_sp_logger.log_info('No new data is available.')
140+
except:
141+
_sp_logger.log_exception(
142+
'An error occured during the data update.', exc_info=True)
143+
144+
def setup_data():
145+
"""Setup data."""
146+
_sp_logger.log_debug('Setting up data...')
147+
148+
from core import GameConfigObj
149+
from memory.manager import manager
150+
from paths import SP_DATA_PATH
151+
152+
import players
153+
players.BaseClient = manager.create_type_from_dict(
154+
'BaseClient',
155+
GameConfigObj(SP_DATA_PATH / 'client' / 'CBaseClient.ini'))
156+
157+
import listeners
158+
listeners.BaseEntityOutput = manager.create_type_from_dict(
159+
'BaseEntityOutput',
160+
GameConfigObj(SP_DATA_PATH / 'entity_output' / 'CBaseEntityOutput.ini'))
161+
162+
try:
163+
_fire_output = listeners.BaseEntityOutput.fire_output
164+
except AttributeError:
165+
from warnings import warn
166+
warn(
167+
'BaseEntityOutput.fire_output not found. '
168+
'OnEntityOutput listener will not fire.'
169+
)
170+
else:
171+
_fire_output.add_pre_hook(listeners._pre_fire_output)
172+
173+
68174
# =============================================================================
69175
# >> CORE SETTINGS
70176
# =============================================================================
@@ -121,24 +227,13 @@ def setup_logging():
121227
# =============================================================================
122228
# >> HOOKS
123229
# =============================================================================
124-
def setup_hooks():
230+
def setup_exception_hooks():
125231
"""Set up hooks."""
126-
_sp_logger.log_debug('Setting up hooks...')
232+
_sp_logger.log_debug('Setting up exception hooks...')
127233

128234
from hooks.exceptions import except_hooks
129235
from hooks.warnings import warning_hooks
130236

131-
# This is added to warn about BaseEntityOutput.fire_output.
132-
# Sending the warning on its initial import will happen prior
133-
# to these hooks being setup.
134-
from listeners._entity_output import _fire_output
135-
if _fire_output is None:
136-
from warnings import warn
137-
warn(
138-
'BaseEntityOutput.fire_output not found. '
139-
'OnEntityOutput listener will not fire.'
140-
)
141-
142237

143238
# =============================================================================
144239
# >> TRANSLATIONS

‎addons/source-python/packages/source-python/auth/manager.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,6 @@
1010
import re
1111
# Importlib
1212
from importlib.machinery import SourceFileLoader
13-
# Site-Package Imports
14-
# Configobj
15-
from configobj import ConfigObj
1613
# Source.Python Imports
1714
# Auth
1815
from auth.base import Backend

‎addons/source-python/packages/source-python/core/settings.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ def __init__(self, infile, *args, **kwargs):
4040
# Import the file
4141
super().__init__(infile, *args, **kwargs)
4242
self._language = None
43+
self.auto_data_update = True
4344

4445
def load(self):
4546
"""Load and update the core settings."""
@@ -87,6 +88,15 @@ def _check_base_settings(self):
8788
self['BASE_SETTINGS'].comments['language'] = _core_strings[
8889
'language'].get_string(self._language).splitlines()
8990

91+
# Auto data update
92+
if 'auto_data_update' not in self['BASE_SETTINGS']:
93+
self['BASE_SETTINGS']['auto_data_update'] = '1'
94+
95+
self.auto_data_update = self['BASE_SETTINGS']['auto_data_update'] == '1'
96+
97+
self['BASE_SETTINGS'].comments['auto_data_update'] = _core_strings[
98+
'auto_data_update'].get_string(self._language).splitlines()
99+
90100
def _check_version_settings(self):
91101
"""Add version settings if they are missing."""
92102
if 'VERSION_SETTINGS' not in self:

‎addons/source-python/packages/source-python/core/update.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# ../core/update.py
2+
3+
"""Provides functions to update Source.Python's data files."""
4+
5+
# =============================================================================
6+
# >> IMPORTS
7+
# =============================================================================
8+
# Python Imports
9+
from zipfile import ZipFile
10+
from urllib.request import urlopen
11+
12+
# Source.Python Imports
13+
# Core
14+
from core import core_logger
15+
# Paths
16+
from paths import DATA_PATH
17+
from paths import SP_DATA_PATH
18+
19+
20+
# =============================================================================
21+
# >> ALL DECLARATION
22+
# =============================================================================
23+
__all__ = (
24+
'CHECKSUM_URL',
25+
'DATA_URL',
26+
'DATA_ZIP_FILE',
27+
'download_latest_data',
28+
'get_latest_data_checksum',
29+
'is_new_data_available',
30+
'unpack_data',
31+
'update_data'
32+
)
33+
34+
35+
# =============================================================================
36+
# >> GLOBAL VARIABLES
37+
# =============================================================================
38+
# Don't use __getattr__ here. 'update' is a method of the _LogInstance class.
39+
update_logger = core_logger['update']
40+
41+
DATA_ZIP_FILE = DATA_PATH / 'source-python-data.zip'
42+
CHECKSUM_URL = 'http://data.sourcepython.com/checksum.txt'
43+
DATA_URL = 'http://data.sourcepython.com/source-python-data.zip'
44+
45+
46+
# =============================================================================
47+
# >> FUNCTIONS
48+
# =============================================================================
49+
def get_latest_data_checksum(timeout=3):
50+
"""Return the MD5 checksum of the latest data from the build server.
51+
52+
:param float timeout:
53+
Number of seconds that need to pass until a timeout occurs.
54+
:rtype: str
55+
"""
56+
with urlopen(CHECKSUM_URL, timeout=timeout) as url:
57+
return url.read().decode()
58+
59+
def download_latest_data(timeout=3):
60+
"""Download the latest data from the build server.
61+
62+
:param float timeout:
63+
Number of seconds that need to pass until a timeout occurs.
64+
"""
65+
update_logger.log_debug('Downloading data to {} ...'.format(DATA_ZIP_FILE))
66+
with urlopen(DATA_URL, timeout=timeout) as url:
67+
data = url.read()
68+
69+
with DATA_ZIP_FILE.open('wb') as f:
70+
f.write(data)
71+
72+
def unpack_data():
73+
"""Unpack ``source-python-data.zip``."""
74+
update_logger.log_debug('Extracting data in {} ...'.format(DATA_PATH))
75+
with ZipFile(DATA_ZIP_FILE) as zip:
76+
zip.extractall(DATA_PATH)
77+
78+
def update_data(timeout=3):
79+
"""Download and unpack the latest data from the build server.
80+
81+
:param float timeout:
82+
Number of seconds that need to pass until a timeout occurs.
83+
"""
84+
download_latest_data(timeout)
85+
if SP_DATA_PATH.isdir():
86+
update_logger.log_debug('Removing {} ...'.format(SP_DATA_PATH))
87+
SP_DATA_PATH.rmtree()
88+
89+
unpack_data()
90+
91+
def is_new_data_available(timeout=3):
92+
"""Return ``True`` if new data is available.
93+
94+
:param float timeout:
95+
Number of seconds that need to pass until a timeout occurs.
96+
:rtype: bool
97+
"""
98+
if not DATA_ZIP_FILE.isfile():
99+
return True
100+
101+
return DATA_ZIP_FILE.read_hexhash('md5') != get_latest_data_checksum(timeout)

‎addons/source-python/packages/source-python/listeners/__init__.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
# Core
1515
from core import AutoUnload
1616
from core import SOURCE_ENGINE
17+
from core import PLATFORM
1718
from core.settings import _core_settings
1819
from core.version import get_last_successful_build_number
1920
from core.version import is_unversioned
@@ -23,6 +24,8 @@
2324
from cvars import cvar
2425
# Engines
2526
from engines.server import server_game_dll
27+
from entities.datamaps import Variant
28+
from entities.helpers import find_output_name
2629
# Memory
2730
from memory import get_virtual_function
2831
# Players
@@ -63,8 +66,6 @@
6366
from _listeners import on_server_activate_listener_manager
6467
from _listeners import on_tick_listener_manager
6568
from _listeners import on_server_output_listener_manager
66-
# Entity output
67-
from listeners._entity_output import on_entity_output_listener_manager
6869

6970

7071
# =============================================================================
@@ -154,6 +155,7 @@
154155
on_level_end_listener_manager = ListenerManager()
155156
on_player_run_command_listener_manager = ListenerManager()
156157
on_button_state_changed_listener_manager = ListenerManager()
158+
on_entity_output_listener_manager = ListenerManager()
157159

158160
_check_for_update = ConVar(
159161
'sp_check_for_update',
@@ -536,6 +538,47 @@ def _pre_call_global_change_callbacks(args):
536538
on_convar_changed_listener_manager.notify(convar, old_value)
537539

538540

541+
def _pre_fire_output(args):
542+
"""Called when an output is about to be fired."""
543+
if not on_entity_output_listener_manager:
544+
return
545+
546+
# Windows is a bit weird: the function takes 4 additional arguments...
547+
if PLATFORM == 'windows':
548+
args = (args[0],) + tuple(args)[5:]
549+
550+
caller_ptr = args[3]
551+
if not caller_ptr:
552+
# If we don't know the caller, we won't be able to retrieve the
553+
# output name
554+
return
555+
556+
# Done here to fix cyclic import...
557+
from entities.entity import BaseEntity
558+
caller = make_object(BaseEntity, caller_ptr)
559+
output_name = find_output_name(caller, args[0])
560+
if output_name is None:
561+
return None
562+
563+
# Done here to fix cyclic import...
564+
from entities.entity import Entity
565+
if caller.is_networked():
566+
caller = make_object(Entity, caller_ptr)
567+
568+
value_ptr = args[1]
569+
value = (value_ptr or None) and make_object(Variant, value_ptr)
570+
571+
activator_ptr = args[2]
572+
activator = ((activator_ptr or None) and make_object(
573+
BaseEntity, activator_ptr))
574+
if activator is not None and activator.is_networked():
575+
activator = make_object(Entity, activator_ptr)
576+
577+
delay = args[4]
578+
on_entity_output_listener_manager.notify(
579+
output_name, activator, caller, value, delay)
580+
581+
539582
# ============================================================================
540583
# >> Fix for issue #181.
541584
# ============================================================================

0 commit comments

Comments
 (0)