forked from Imagelibrary/littlefs
Fixed inotify race conditions and fd leak in scripts
Since we were only registering our inotify reader after the previous operation completed, it was easy to miss modifications that happened faster than our scripts. Since our scripts are in Python, this happened quite often and made it hard to trust the current state of scripts with --keep-open, sort of defeating the purpose of --keep-open... I think previously this race condition wasn't avoided because of the potential to loop indefinitely if --keep-open referenced a file that the script itself modified, but it's up to the user to avoid this if it is an issue. --- Also while fixing this, I noticed our use of the inotify_simple library was leaking file descriptors everywhere! I just wasn't closing any inotify objects at all. A bit concerning since scripts with --keep-open can be quite long lived...
This commit is contained in:
Christopher Haster
@@ -140,9 +140,14 @@ def openio(path, mode='r', buffering=-1):
|
||||
else:
|
||||
return open(path, mode, buffering)
|
||||
|
||||
def inotifywait(paths):
|
||||
if inotify_simple is None:
|
||||
Inotify = None
|
||||
else:
|
||||
class Inotify(inotify_simple.INotify):
|
||||
def __init__(self, paths):
|
||||
super().__init__()
|
||||
|
||||
# wait for interesting events
|
||||
inotify = inotify_simple.INotify()
|
||||
flags = (inotify_simple.flags.ATTRIB
|
||||
| inotify_simple.flags.CREATE
|
||||
| inotify_simple.flags.DELETE
|
||||
@@ -156,14 +161,11 @@ def inotifywait(paths):
|
||||
for path in paths:
|
||||
if os.path.isdir(path):
|
||||
for dir, _, files in os.walk(path):
|
||||
inotify.add_watch(dir, flags)
|
||||
self.add_watch(dir, flags)
|
||||
for f in files:
|
||||
inotify.add_watch(os.path.join(dir, f), flags)
|
||||
self.add_watch(os.path.join(dir, f), flags)
|
||||
else:
|
||||
inotify.add_watch(path, flags)
|
||||
|
||||
# wait for event
|
||||
inotify.read()
|
||||
self.add_watch(path, flags)
|
||||
|
||||
class LinesIO:
|
||||
def __init__(self, maxlen=None):
|
||||
@@ -1392,6 +1394,11 @@ def main(csv_paths, *,
|
||||
if keep_open:
|
||||
try:
|
||||
while True:
|
||||
# register inotify before running the command, this avoids
|
||||
# modification race conditions
|
||||
if keep_open and Inotify:
|
||||
inotify = Inotify(csv_paths)
|
||||
|
||||
if cat:
|
||||
draw(sys.stdout)
|
||||
else:
|
||||
@@ -1400,11 +1407,12 @@ def main(csv_paths, *,
|
||||
ring.draw()
|
||||
|
||||
# try to inotifywait
|
||||
if inotify_simple is not None:
|
||||
if keep_open and Inotify:
|
||||
ptime = time.time()
|
||||
inotifywait(csv_paths)
|
||||
# sleep for a minimum amount of time, this helps issues
|
||||
# around rapidly updating files
|
||||
inotify.read()
|
||||
inotify.close()
|
||||
# sleep for a minimum amount of time, this helps reduce
|
||||
# flicker issues
|
||||
time.sleep(max(0, (sleep or 0.01) - (time.time()-ptime)))
|
||||
else:
|
||||
time.sleep(sleep or 0.1)
|
||||
|
||||
@@ -40,9 +40,14 @@ def openio(path, mode='r', buffering=-1):
|
||||
else:
|
||||
return open(path, mode, buffering)
|
||||
|
||||
def inotifywait(paths):
|
||||
if inotify_simple is None:
|
||||
Inotify = None
|
||||
else:
|
||||
class Inotify(inotify_simple.INotify):
|
||||
def __init__(self, paths):
|
||||
super().__init__()
|
||||
|
||||
# wait for interesting events
|
||||
inotify = inotify_simple.INotify()
|
||||
flags = (inotify_simple.flags.ATTRIB
|
||||
| inotify_simple.flags.CREATE
|
||||
| inotify_simple.flags.DELETE
|
||||
@@ -56,14 +61,11 @@ def inotifywait(paths):
|
||||
for path in paths:
|
||||
if os.path.isdir(path):
|
||||
for dir, _, files in os.walk(path):
|
||||
inotify.add_watch(dir, flags)
|
||||
self.add_watch(dir, flags)
|
||||
for f in files:
|
||||
inotify.add_watch(os.path.join(dir, f), flags)
|
||||
self.add_watch(os.path.join(dir, f), flags)
|
||||
else:
|
||||
inotify.add_watch(path, flags)
|
||||
|
||||
# wait for event
|
||||
inotify.read()
|
||||
self.add_watch(path, flags)
|
||||
|
||||
class LinesIO:
|
||||
def __init__(self, maxlen=None):
|
||||
@@ -147,6 +149,21 @@ def main(command, *,
|
||||
if keep_open_paths and not keep_open:
|
||||
keep_open = True
|
||||
|
||||
# figure out the keep_open paths
|
||||
if keep_open and inotify_simple is not None:
|
||||
if keep_open_paths:
|
||||
keep_open_paths = set(keep_open_paths)
|
||||
else:
|
||||
# guess inotify paths from command
|
||||
keep_open_paths = set()
|
||||
for p in command:
|
||||
for p in {
|
||||
p,
|
||||
re.sub('^-.', '', p),
|
||||
re.sub('^--[^=]+=', '', p)}:
|
||||
if p and os.path.exists(p):
|
||||
paths.add(p)
|
||||
|
||||
returncode = 0
|
||||
try:
|
||||
while True:
|
||||
@@ -157,6 +174,11 @@ def main(command, *,
|
||||
ring = LinesIO(lines)
|
||||
|
||||
try:
|
||||
# register inotify before running the command, this avoids
|
||||
# modification race conditions
|
||||
if keep_open and Inotify:
|
||||
inotify = Inotify(keep_open_paths)
|
||||
|
||||
# run the command under a pseudoterminal
|
||||
mpty, spty = pty.openpty()
|
||||
|
||||
@@ -204,24 +226,14 @@ def main(command, *,
|
||||
pass
|
||||
|
||||
# try to inotifywait
|
||||
if keep_open and inotify_simple is not None:
|
||||
if keep_open_paths:
|
||||
paths = set(keep_open_paths)
|
||||
else:
|
||||
# guess inotify paths from command
|
||||
paths = set()
|
||||
for p in command:
|
||||
for p in {
|
||||
p,
|
||||
re.sub('^-.', '', p),
|
||||
re.sub('^--[^=]+=', '', p)}:
|
||||
if p and os.path.exists(p):
|
||||
paths.add(p)
|
||||
if keep_open and Inotify:
|
||||
ptime = time.time()
|
||||
inotifywait(paths)
|
||||
# sleep for a minimum amount of time, this helps issues around
|
||||
# rapidly updating files
|
||||
time.sleep(max(0, (sleep or 0.1) - (time.time()-ptime)))
|
||||
inotify.read()
|
||||
inotify.close()
|
||||
# sleep for a minimum amount of time, this helps reduce
|
||||
# flicker issues
|
||||
time.sleep(max(0, (sleep or 0.01) - (time.time()-ptime)))
|
||||
# or sleep
|
||||
else:
|
||||
time.sleep(sleep or 0.1)
|
||||
except KeyboardInterrupt:
|
||||
|
||||
Reference in New Issue
Block a user