Commit 07df696e authored by Martin van Es's avatar Martin van Es
Browse files

Better metadata reload

parent 7379cf3e
#!/usr/bin/env python
from utils import read_config, Resource, Server
from utils import read_config, Resource
from flask import Flask, Response
config = read_config('mdserver.yaml')
app = Flask(__name__)
server = Server()
server = {}
@app.route('/<domain>/entities/<path:entity_id>',
......@@ -29,8 +29,6 @@ for domain, values in config.items():
location = values['metadir']
signer = values['signer']
server[domain] = Resource(location, signer)
server.add_watch(domain, location)
if __name__ == "__main__":
app.run(host='127.0.0.1', port=5001, debug=False)
......@@ -9,9 +9,6 @@ import yaml
import pyinotify
from signers import Signers
# watch_list = {}
# watch_manager = pyinotify.WatchManager()
def read_config(config):
with open(config) as f:
......@@ -33,28 +30,69 @@ class Entity:
self.cache_duration = 0
class EventProcessor(pyinotify.ProcessEvent):
def __init__(self, resource):
self.resource = resource
def process_default(self, event):
if event.maskname in ["IN_DELETE",
"IN_MOVED_FROM",
"IN_MOVED_TO"]:
self.resource.walk_location(event.path)
else:
self.resource.read_metadata(event.pathname)
class Resource:
watch_list = {}
def __init__(self, location, signer):
self.idps = {}
self.mdfiles = {}
self.signer = Signers(signer)
self.md_watcher = pyinotify.WatchManager()
self.md_events = pyinotify.ThreadedNotifier(self.md_watcher,
EventProcessor(self))
self.md_events.start()
self.walk_location(location)
def walk_location(self, location):
old_mdfiles = self.mdfiles.copy()
location = os.path.realpath(location)
self.md_watcher.add_watch(location, pyinotify.IN_CLOSE_WRITE
| pyinotify.IN_DELETE
| pyinotify.IN_MOVED_FROM
| pyinotify.IN_MOVED_TO)
files = os.listdir(location)
for file in files:
mdfile = os.path.realpath(os.path.join(location, file))
if os.path.isfile(mdfile):
self.mdfiles[mdfile] = []
self._read_metadata(mdfile)
def _read_metadata(self, mdfile):
old_mdfiles.pop(mdfile, None)
self.read_metadata(mdfile)
for mdf, idps in old_mdfiles.items():
print("\n--- REMOVE METADATA --")
print(mdf)
for idp in idps:
print(f" {{sha1}}{idp}")
self.idps.pop(idp, None)
self.__dict__.pop(idp, None)
def read_metadata(self, mdfile):
print("\n--- READ METADATA --")
print(mdfile)
found = 0
removed = 0
old_idps = self.mdfiles[mdfile].copy()
root = ET.ElementTree(file=mdfile).getroot()
old_idps = self.mdfiles.get(mdfile, []).copy()
mdfiles = []
try:
root = ET.ElementTree(file=mdfile).getroot()
except Exception:
print("Invalid metadata")
return
ns = root.nsmap.copy()
ns['xml'] = 'http://www.w3.org/XML/1998/namespace'
validUntil = root.get('validUntil')
......@@ -76,15 +114,16 @@ class Resource:
valid_until)
self.idps[sha1] = entity
self.__dict__.pop(sha1, None)
if sha1 in self.mdfiles[mdfile]:
self.mdfiles[mdfile].remove(sha1)
if sha1 in old_idps:
old_idps.remove(sha1)
mdfiles.append(sha1)
found += 1
for idp in old_idps:
self.idps.pop(idp, None)
self.__dict__.pop(idp, None)
removed += 1
self.mdfiles[mdfile] = self.idps.keys()
self.mdfiles[mdfile] = list(mdfiles)
print(f"Found: {found} entities")
print(f"Removed: {removed} entities")
......@@ -125,38 +164,3 @@ class Resource:
print(f"serve {sha1}")
return data
class EventProcessor(pyinotify.ProcessEvent):
def __init__(self, server):
self.server = server
def process_IN_CLOSE_WRITE(self, event):
self.server.process(event.path)
class Server:
watch_list = {}
def __init__(self):
self.watch_manager = pyinotify.WatchManager()
self.event_notifier = pyinotify.ThreadedNotifier(self.watch_manager,
EventProcessor(self))
self.event_notifier.start()
def add_watch(self, domain, location):
self.watch_list[location] = domain
self.watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
def process(self, location):
domain = self.watch_list[location]
print(f"Notify {domain} {location}")
self.__dict__[domain].walk_location(location)
def __setitem__(self, domain, resource):
self.__dict__[domain] = resource
# watch_list[location] = domain
# watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
def __getitem__(self, domain):
return self.__dict__[domain]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment