Skip to content

Commit 161b4fd

Browse files
DuctTape42DuctTape42
committed
Watcher is now aware of which platform you're on. Adds heuristics for Windows, Linux, and a generic Poller.
Mac OS just mirrors Linux (hopefully that works). Windows tests to see if the file is openable (works on local files to see if they're still in use before triggering an update)
1 parent c722cc6 commit 161b4fd

2 files changed

Lines changed: 97 additions & 3 deletions

File tree

‎scripts/stash-watcher/defaults.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@ Cooldown = 300
1919
#A list of file extensions to watch. If this is omitted, it uses the extensions that are defined
2020
#in your Stash library (for videos, images, and galleries)
2121
Extensions =
22+
#If this is set to a non-zero numeric value, this forces the use of polling to
23+
#determine file system changes. If it is left blank, then the OS appropriate
24+
#mechanism is used. This is much less efficient than the OS mechanism, so it
25+
#should be used with care. The docs claim that this is required to watch SMB
26+
#shares, though in my testing I could watch them on Windows with the regular
27+
#WindowsApiObserver
28+
PollInterval=
29+
#This enables debug logging
30+
Debug=
2231

2332
#Options for the Stash Scan. Stash defaults to everything disabled, so this is the default
2433
#Generate options that match up with what we can do in Scan

‎scripts/stash-watcher/watcher.py‎

Lines changed: 88 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@
55
import os
66
from threading import Lock, Condition
77
from watchdog.observers import Observer
8+
from watchdog.observers.polling import PollingObserver
89
from watchdog.events import PatternMatchingEventHandler
910
from stashapi.stashapp import StashInterface
1011
import logging
12+
import sys
13+
from enum import Enum
14+
15+
#the type of watcher being used; controls how to interpret the events
16+
WatcherType = Enum('WatcherType', ['INOTIFY', 'WINDOWS', 'POLLING', 'KQUEUE'])
1117

1218
#Setup logger
1319
logger = logging.getLogger("stash-watcher")
@@ -24,6 +30,10 @@
2430

2531
modifiedFiles = {}
2632

33+
34+
currentWatcherType = None
35+
36+
2737
def log(msg):
2838
logger.info(msg)
2939

@@ -32,12 +42,57 @@ def debug(msg):
3242

3343
def handleEvent(event):
3444
global shouldUpdate
45+
global currentWatcherType
3546
debug("========EVENT========")
3647
debug(str(event))
3748
#log(modifiedFiles)
3849
#Record if the file was modified. When a file is closed, see if it was modified. If so, trigger
3950
shouldTrigger = False
40-
if event.is_directory == False:
51+
52+
if event.is_directory == True:
53+
return
54+
#Depending on the watcher type, we have to handle these events differently
55+
if currentWatcherType == WatcherType.WINDOWS:
56+
#On windows here's what happens:
57+
# File moved into a watched directory - Created Event
58+
# File moved out of a watched directory - Deleted Event
59+
# Moved within a watched directory (src and dst in watched directory) - Moved event
60+
61+
# echo blah > foo.mp4 - Created then Modified
62+
# copying a small file - Created then Modified
63+
# copying a large file - Created then two (or more) Modified events (appears to be one when the file is created and another when it's finished)
64+
65+
#It looks like you can get an optional Created Event and then
66+
#either one or two Modified events. You can also get Moved events
67+
68+
#For local files on Windows, they can't be opened if they're currently
69+
#being written to. Therefore, every time we get an event, attempt to
70+
#open the file. If we're successful, assume the write is finished and
71+
#trigger the update. Otherwise wait until the next event and try again
72+
if event.event_type == "created" or event.event_type == "modified":
73+
try:
74+
with open(event.src_path) as file:
75+
debug("Successfully opened file; triggering")
76+
shouldTrigger = True
77+
except:
78+
pass
79+
80+
if event.event_type == "moved":
81+
shouldTrigger = True
82+
elif currentWatcherType == WatcherType.POLLING:
83+
#Every interval you get 1 event per changed file
84+
# - If the file was not present in the previous poll, then Created
85+
# - If the file was present and has a new size, then Modified
86+
# - If the file was moved within the directory, then Moved
87+
# - If the file is gone, then deleted
88+
#
89+
# For now, just trigger on the created event. In the future, create
90+
# a timer at 2x polling interval. Reschedule the timer on each event
91+
# when it fires, trigger the update.
92+
if event.event_type == "moved" or event.event_type == "created":
93+
shouldTrigger = True
94+
#Until someone tests this on mac, just do what INOTIFY does
95+
elif currentWatcherType == WatcherType.INOTIFY or currentWatcherType == WatcherType.KQUEUE:
4196
if event.event_type == "modified":
4297
modifiedFiles[event.src_path] = 1
4398
#These are for files being copied into the target
@@ -50,6 +105,9 @@ def handleEvent(event):
50105
#moved out of a watched directory
51106
elif event.event_type == "moved":
52107
shouldTrigger = True
108+
else:
109+
print("Unknown watcher type " + str(currentWatcherType))
110+
sys.exit(1)
53111

54112
#Trigger the update
55113
if shouldTrigger:
@@ -59,8 +117,9 @@ def handleEvent(event):
59117
signal.notify()
60118

61119

62-
def main(stash, scanFlags, paths, extensions, timeout):
120+
def main(stash, scanFlags, paths, extensions, timeout, pollInterval):
63121
global shouldUpdate
122+
global currentWatcherType
64123

65124
if len(extensions) == 1 and extensions[0] == "*":
66125
patterns = ["*"]
@@ -69,6 +128,21 @@ def main(stash, scanFlags, paths, extensions, timeout):
69128
eventHandler = PatternMatchingEventHandler(patterns, None, False, True)
70129
eventHandler.on_any_event = handleEvent
71130
observer = Observer()
131+
observerName = type(observer).__name__
132+
if pollInterval != None and pollInterval > 0:
133+
currentWatcherType = WatcherType.POLLING
134+
observer = PollingObserver()
135+
elif observerName == "WindowsApiObserver":
136+
currentWatcherType = WatcherType.WINDOWS
137+
elif observerName == "KqueueObserver":
138+
currentWatcherType = WatcherType.KQUEUE
139+
elif observerName == "InotifyObserver":
140+
currentWatcherType = WatcherType.INOTIFY
141+
else:
142+
print("Unknown watcher type " + str(observer))
143+
sys.exit(1)
144+
145+
debug(str(observer))
72146
for path in paths:
73147
observer.schedule(eventHandler, path, recursive=True)
74148
observer.start()
@@ -152,7 +226,18 @@ def parseConfig(path):
152226
stashConfig = stash.graphql_configuration()
153227
extensions = stashConfig['general']['videoExtensions'] + stashConfig['general']['imageExtensions'] + stashConfig['general']['galleryExtensions']
154228

155-
main(stash, scanFlags, paths, extensions, timeout)
229+
pollIntervalStr = config.get('Config', 'PollInterval')
230+
if pollIntervalStr:
231+
pollInterval = int(pollIntervalStr)
232+
else:
233+
pollInterval = None
234+
235+
if config.get('Config', 'Debug') == "true":
236+
logger.setLevel(logging.DEBUG)
237+
ch.setLevel(logging.DEBUG)
238+
239+
240+
main(stash, scanFlags, paths, extensions, timeout, pollInterval)
156241

157242

158243

0 commit comments

Comments
 (0)