Commit 07df696e authored by Martin van Es's avatar Martin van Es
Browse files

Better metadata reload

parent 7379cf3e
#!/usr/bin/env python #!/usr/bin/env python
from utils import read_config, Resource, Server from utils import read_config, Resource
from flask import Flask, Response from flask import Flask, Response
config = read_config('mdserver.yaml') config = read_config('mdserver.yaml')
app = Flask(__name__) app = Flask(__name__)
server = Server() server = {}
@app.route('/<domain>/entities/<path:entity_id>', @app.route('/<domain>/entities/<path:entity_id>',
...@@ -29,8 +29,6 @@ for domain, values in config.items(): ...@@ -29,8 +29,6 @@ for domain, values in config.items():
location = values['metadir'] location = values['metadir']
signer = values['signer'] signer = values['signer']
server[domain] = Resource(location, signer) server[domain] = Resource(location, signer)
server.add_watch(domain, location)
if __name__ == "__main__": if __name__ == "__main__":
app.run(host='127.0.0.1', port=5001, debug=False) app.run(host='127.0.0.1', port=5001, debug=False)
...@@ -9,9 +9,6 @@ import yaml ...@@ -9,9 +9,6 @@ import yaml
import pyinotify import pyinotify
from signers import Signers from signers import Signers
# watch_list = {}
# watch_manager = pyinotify.WatchManager()
def read_config(config): def read_config(config):
with open(config) as f: with open(config) as f:
...@@ -33,28 +30,69 @@ class Entity: ...@@ -33,28 +30,69 @@ class Entity:
self.cache_duration = 0 self.cache_duration = 0
class EventProcessor(pyinotify.ProcessEvent):
def __init__(self, resource):
self.resource = resource
def process_default(self, event):
if event.maskname in ["IN_DELETE",
"IN_MOVED_FROM",
"IN_MOVED_TO"]:
self.resource.walk_location(event.path)
else:
self.resource.read_metadata(event.pathname)
class Resource: class Resource:
watch_list = {}
def __init__(self, location, signer): def __init__(self, location, signer):
self.idps = {} self.idps = {}
self.mdfiles = {} self.mdfiles = {}
self.signer = Signers(signer) self.signer = Signers(signer)
self.md_watcher = pyinotify.WatchManager()
self.md_events = pyinotify.ThreadedNotifier(self.md_watcher,
EventProcessor(self))
self.md_events.start()
self.walk_location(location) self.walk_location(location)
def walk_location(self, location): def walk_location(self, location):
old_mdfiles = self.mdfiles.copy()
location = os.path.realpath(location)
self.md_watcher.add_watch(location, pyinotify.IN_CLOSE_WRITE
| pyinotify.IN_DELETE
| pyinotify.IN_MOVED_FROM
| pyinotify.IN_MOVED_TO)
files = os.listdir(location) files = os.listdir(location)
for file in files: for file in files:
mdfile = os.path.realpath(os.path.join(location, file)) mdfile = os.path.realpath(os.path.join(location, file))
if os.path.isfile(mdfile): if os.path.isfile(mdfile):
self.mdfiles[mdfile] = [] old_mdfiles.pop(mdfile, None)
self._read_metadata(mdfile) self.read_metadata(mdfile)
def _read_metadata(self, mdfile): for mdf, idps in old_mdfiles.items():
print("\n--- REMOVE METADATA --")
print(mdf)
for idp in idps:
print(f" {{sha1}}{idp}")
self.idps.pop(idp, None)
self.__dict__.pop(idp, None)
def read_metadata(self, mdfile):
print("\n--- READ METADATA --") print("\n--- READ METADATA --")
print(mdfile) print(mdfile)
found = 0 found = 0
removed = 0 removed = 0
old_idps = self.mdfiles[mdfile].copy() old_idps = self.mdfiles.get(mdfile, []).copy()
root = ET.ElementTree(file=mdfile).getroot() mdfiles = []
try:
root = ET.ElementTree(file=mdfile).getroot()
except Exception:
print("Invalid metadata")
return
ns = root.nsmap.copy() ns = root.nsmap.copy()
ns['xml'] = 'http://www.w3.org/XML/1998/namespace' ns['xml'] = 'http://www.w3.org/XML/1998/namespace'
validUntil = root.get('validUntil') validUntil = root.get('validUntil')
...@@ -76,15 +114,16 @@ class Resource: ...@@ -76,15 +114,16 @@ class Resource:
valid_until) valid_until)
self.idps[sha1] = entity self.idps[sha1] = entity
self.__dict__.pop(sha1, None) self.__dict__.pop(sha1, None)
if sha1 in self.mdfiles[mdfile]: if sha1 in old_idps:
self.mdfiles[mdfile].remove(sha1) old_idps.remove(sha1)
mdfiles.append(sha1)
found += 1 found += 1
for idp in old_idps: for idp in old_idps:
self.idps.pop(idp, None) self.idps.pop(idp, None)
self.__dict__.pop(idp, None) self.__dict__.pop(idp, None)
removed += 1 removed += 1
self.mdfiles[mdfile] = self.idps.keys() self.mdfiles[mdfile] = list(mdfiles)
print(f"Found: {found} entities") print(f"Found: {found} entities")
print(f"Removed: {removed} entities") print(f"Removed: {removed} entities")
...@@ -125,38 +164,3 @@ class Resource: ...@@ -125,38 +164,3 @@ class Resource:
print(f"serve {sha1}") print(f"serve {sha1}")
return data return data
class EventProcessor(pyinotify.ProcessEvent):
def __init__(self, server):
self.server = server
def process_IN_CLOSE_WRITE(self, event):
self.server.process(event.path)
class Server:
watch_list = {}
def __init__(self):
self.watch_manager = pyinotify.WatchManager()
self.event_notifier = pyinotify.ThreadedNotifier(self.watch_manager,
EventProcessor(self))
self.event_notifier.start()
def add_watch(self, domain, location):
self.watch_list[location] = domain
self.watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
def process(self, location):
domain = self.watch_list[location]
print(f"Notify {domain} {location}")
self.__dict__[domain].walk_location(location)
def __setitem__(self, domain, resource):
self.__dict__[domain] = resource
# watch_list[location] = domain
# watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
def __getitem__(self, domain):
return self.__dict__[domain]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment