Commit 6f0500cf authored by Marco Malavolti's avatar Marco Malavolti
Browse files

Added multiprocessing execution

parent 0cecf43b
......@@ -3,11 +3,17 @@
* `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev`
* `python3 -m pip install --user --upgrade pip virtualenv`
* `python3 -m venv eccs2venv`
* `source eccs2venv/bin/activate` (`deactivate` di exit Virtualenv)
* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv)
* `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
* `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
* `cd eccs2 ; ./eccs2.py`
# API Development Server
* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
* `sudo a2enmod wsgi`
* `cd ~/eccs2 ; ./api.py`
# API
* `/eccs/test` (Trivial Test)
......@@ -23,8 +29,4 @@
* 'Excluded'
* /eccs/eccsresults (Return the results of the last check ready for ECCS Gui)
# API Development Server
* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
* `sudo a2enmod wsgi`
* `cd ~/eccs2 ; ./api.py`
......@@ -8,6 +8,7 @@ from pathlib import PurePath
import logging
from logging.handlers import RotatingFileHandler
import re
import eccs2properties
app = Flask(__name__)
......@@ -67,7 +68,7 @@ class Checks(Resource):
def get(self):
app.logger.info("Request 'Checks'")
file_path = "logs/eccs2checks_2020-02-22.log"
file_path = "logs/eccs2checks_%s.log" % eccs2properties.day
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
......@@ -142,7 +143,6 @@ class Checks(Resource):
# Build Email Addresses Link for ECCS2 Web Gui
def buildEmailAddress(listContacts):
listCtcs = listContacts.split(",")
hrefList = []
......@@ -155,7 +155,7 @@ class EccsResults(Resource):
def get(self):
app.logger.info("Request 'EccsResults'")
file_path = "logs/eccs2_2020-03-01.log"
file_path = "logs/eccs2_%s.log" % eccs2properties.day
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
......@@ -305,10 +305,15 @@ class EccsResults(Resource):
else:
return jsonify(result)
# Run check for a specific IDP
# <idpdisc:DiscoveryResponse Location>?entityID=<IDP_ENITIYID>&target=<DESTINATION_RESOURCE_URL> (tutto url encoded)
#class RunCheck(Resource):
# def get(self):
api.add_resource(Test, '/eccs/test') # Route_1
api.add_resource(Checks, '/eccs/checks') # Route_2
api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3
#api.add_resource(RunCheck, '/eccs/runcheck') # Route_4
if __name__ == '__main__':
......
#!/usr/bin/env python3
#!/usr/bin/env python3.8
from datetime import date
import argparse
import json
import logging
import time
import os
import eccs2properties
import psutil
import signal
import re
import requests
from datetime import date
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
"""
......@@ -31,47 +47,61 @@ def getIdpListFromUrl():
def getIdpListFromFile():
import json
with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
#with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
with open('federation_idps.txt','r',encoding='utf-8') as f:
json_data = json.loads(f.read())
return json_data
def checkIdP(sp,idp,logger):
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
import re
# Disable SSL requests warning messages
requests.packages.urllib3.disable_warnings()
# Configure Web-driver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--ignore-certificate-errors')
# driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=./selenium_chromedriver.log'])
#driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
#driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
# Configure timeouts: 45 sec
driver.set_page_load_timeout(45)
driver.set_script_timeout(45)
# Configure timeouts: 30 sec
driver.set_page_load_timeout(30)
driver.set_script_timeout(30)
# Configure Blacklists
federation_blacklist = ['http://www.surfconext.nl/','https://www.wayf.dk','http://feide.no/']
entities_blacklist = ['https://idp.eie.gr/idp/shibboleth','https://gn-vho.grnet.gr/idp/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://idp.fraunhofer.de/idp/shibboleth','https://login.hs-owl.de/nidp/saml2/metadata','https://idp.dfn-cert.de/idp/shibboleth']
federation_blacklist = [
'http://www.surfconext.nl/',
'https://www.wayf.dk',
'http://feide.no/'
]
entities_blacklist = [
'https://idp.eie.gr/idp/shibboleth',
'https://gn-vho.grnet.gr/idp/shibboleth',
'https://wtc.tu-chemnitz.de/shibboleth',
'https://wtc.tu-chemnitz.de/shibboleth',
'https://idp.fraunhofer.de/idp/shibboleth',
'https://login.hs-owl.de/nidp/saml2/metadata',
'https://idp.dfn-cert.de/idp/shibboleth'
]
if (idp['entityID'] in entities_blacklist):
logger.info("%s;%s;IdP excluded from checks" % (idp['entityID'],sp))
if (idp['registrationAuthority'] in federation_blacklist):
logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
driver.close()
driver.quit()
return "DISABLED"
if (idp['registrationAuthority'] in federation_blacklist):
logger.info("%s;%s;Federation excluded from checks" % (idp['entityID'],sp))
if (idp['entityID'] in entities_blacklist):
logger.info("%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp))
driver.close()
driver.quit()
return "DISABLED"
# Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check
try:
driver.get(sp)
......@@ -79,14 +109,17 @@ def checkIdP(sp,idp,logger):
driver.find_element_by_id("username")
driver.find_element_by_id("password")
except NoSuchElementException as e:
pass
except TimeoutException as e:
logger.info("%s;%s;TIMEOUT" % (idp['entityID'],sp))
driver.delete_all_cookies()
print("TIMEOUT - driver.current_url: %s" % (driver.current_url))
status_code = requests.get(driver.current_url, verify=False).status_code
logger.info("%s;%s;%s;TIMEOUT" % (idp['entityID'],sp,status_code))
driver.close()
driver.quit()
return "TIMEOUT"
except NoSuchElementException as e:
driver.delete_all_cookies()
pass
pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
......@@ -98,17 +131,26 @@ def checkIdP(sp,idp,logger):
password_found = re.search(pattern_password,driver.page_source, re.I)
if(metadata_not_found):
logger.info("%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp))
#print("MD-NOT-FOUND - driver.current_url: %s" % (driver.current_url))
status_code = requests.get(driver.current_url, verify=False).status_code
logger.info("%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code))
driver.delete_all_cookies()
driver.close()
driver.quit()
return "No-eduGAIN-Metadata"
elif not username_found and not password_found:
logger.info("%s;%s;Invalid-Form" % (idp['entityID'],sp))
elif not username_found or not password_found:
#print("INVALID-FORM - entityID: %s, sp: %s, driver.current_url: %s" % (idp['entityID'],sp,driver.current_url))
status_code = requests.get(driver.current_url, verify=False).status_code
logger.info("%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code))
driver.delete_all_cookies()
driver.close()
driver.quit()
return "Invalid Form"
else:
logger.info("%s;%s;OK" % (idp['entityID'],sp))
#print("MD-FOUND - driver.current_url: %s" % (driver.current_url))
status_code = requests.get(driver.current_url, verify=False).status_code
logger.info("%s;%s;%s;OK" % (idp['entityID'],sp,status_code))
driver.delete_all_cookies()
driver.close()
driver.quit()
return "OK"
......@@ -118,7 +160,7 @@ def checkIdP(sp,idp,logger):
def getLogger(filename,log_level="DEBUG",path="./"):
logger = logging.getLogger(filename)
ch = logging.FileHandler(path+filename,'w','utf-8')
ch = logging.FileHandler(path+filename,'a','utf-8')
if (log_level == "DEBUG"):
logger.setLevel(logging.DEBUG)
......@@ -154,23 +196,11 @@ def getIdPContacts(idp,contactType):
return ctcList
# MAIN
if __name__=="__main__":
day = date.today().isoformat()
eccs2log = getLogger("logs/eccs2_"+day+".log","INFO")
eccs2checksLog = getLogger("logs/eccs2checks_"+day+".log","INFO")
sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
#listIdPs = getIdpListFromUrl()
listIdPs = getIdpListFromFile()
for idp in listIdPs:
def checkIdp(idp,sps,eccs2log,eccs2checksLog):
result = []
for sp in sps:
result.append(checkIdP(sp,idp,eccs2checksLog))
resultCheck = checkIdP(sp,idp,eccs2checksLog)
result.append(resultCheck)
listTechContacts = getIdPContacts(idp,'technical')
listSuppContacts = getIdPContacts(idp,'support')
......@@ -216,3 +246,21 @@ if __name__=="__main__":
result[0],
sps[1],
result[1]))
# MAIN
if __name__=="__main__":
eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
#sps = ["https://attribute-viewer.aai.switch.ch/eds/", "https://attribute-viewer.aai.switch.ch/eds/"]
parser = argparse.ArgumentParser(description='Checks if the input IdP consumed correctly eduGAIN metadata by accessing two different SPs')
parser.add_argument("idpJson", metavar="idpJson", nargs=1, help="An IdP in Json format")
args = parser.parse_args()
idp = json.loads(args.idpJson[0])
checkIdp(idp,sps,eccs2log,eccs2checksLog)
#!/usr/bin/env python3.8
import asyncio
import eccs2properties
import json
import sys
import time
from subprocess import Popen,PIPE
def getIdPs():
import certifi
import urllib3
import json
manager = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
url = "https://technical.edugain.org/api.php?action=list_eccs_idps"
idp_json = manager.request('GET', url)
idp_dict = json.loads(idp_json.data.decode('utf-8'))
idp_list = []
#federation = input("Insert the registrationAuthority: ")
federation = "http://www.idem.garr.it/"
for idp in idp_dict:
if (idp['registrationAuthority'] == federation):
idp_list.append(idp)
return json.dumps(idp_list)
def getIdpListFromFile():
import json
#with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
with open('federation_idps.txt','r',encoding='utf-8') as f:
json_data = json.loads(f.read())
return json_data
async def run(name,queue,stdout_file,stderr_file):
while True:
# Get a "cmd item" out of the queue.
cmd = await queue.get()
# Elaborate "cmd" from shell.
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
if stdout:
stdout_file.write(f'[stdout]\n{stdout.decode()}')
if stderr:
stderr_file.write(f'[stderr]\n{stderr.decode()}\n\n[cmd]\n{cmd}')
# Notify the queue that the "work cmd" has been processed.
queue.task_done()
async def main(cmd_list,stdout_file,stderr_file):
# Create a queue that we will use to store our "workload".
queue = asyncio.Queue()
# Put all commands into the queue.
for cmd in cmd_list:
queue.put_nowait(cmd)
# Create worker tasks to process the queue concurrently.
tasks = []
for i in range(30):
task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
tasks.append(task)
# Wait until the queue is fully processed.
started_at = time.monotonic()
await queue.join()
total_slept_for = time.monotonic() - started_at
# Cancel our worker tasks.
for task in tasks:
task.cancel()
# Wait until all worker tasks are cancelled.
await asyncio.gather(*tasks, return_exceptions=True)
# MAIN
if __name__=="__main__":
start = time.time()
'''
data = getIdPs()
f = open('federation_idps.txt', 'w')
f.write(data)
f.close()
'''
stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
idpJsonList = getIdpListFromFile()
num_idps = len(idpJsonList)
cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
proc_list = []
count = 0
while (count < num_idps):
cmd = "".join(cmd_list.pop())
proc_list.append(cmd)
count = count + 1
asyncio.run(main(proc_list,stdout_file,stderr_file))
end = time.time()
print("Time taken in seconds - ", end - start)
......@@ -2,7 +2,7 @@
<html>
<head>
<script type="text/javascript" src="https://code.jquery.com/jquery-3.3.1.js"></script>
<script type="text/javascript" src="https://code.jquery.com/jquery-3.4.1.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js"></script>
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css"/>
......@@ -14,7 +14,7 @@
</head>
<body>
<div class="container">
<table id="example" class="display" style="width:100%">
<table id="eccstable" class="display" style="width:100%">
<thead>
<tr>
<th></th>
......@@ -22,8 +22,7 @@
<th>EntityID</th>
<th>Registration Authority</th>
<th>Technical Contacts</th>
<th>Support Contacts</th>
<th>Date</th>
<th>Check Date</th>
<th>Status</th>
</tr>
</thead>
......
......@@ -12,6 +12,11 @@ function format ( d ) {
'<td>'+d.contacts.technical+'</td>'+
'<td></td>'+
'</tr>'+
'<tr>'+
'<td>Support Contacts:</td>'+
'<td>'+d.contacts.support+'</td>'+
'<td></td>'+
'</tr>'+
'<tr>'+
'<td>SP1:</td>'+
'<td>'+d.sp1.entityID+'</td>'+
......@@ -26,7 +31,7 @@ function format ( d ) {
}
$(document).ready(function() {
var table = $('#example').DataTable( {
var table = $('#eccstable').DataTable( {
"ajax": {
"url": "data.json",
"dataSrc": ""
......@@ -46,11 +51,7 @@ $(document).ready(function() {
{ "data": "registrationAuthority" },
{
"data": "contacts.technical",
"defaultContent": ''
},
{
"data": "contacts.support",
"defaultContent": ''
"defaultContent": ""
},
{ "data": "date" },
{ "data": "status" }
......@@ -70,7 +71,7 @@ $(document).ready(function() {
} );
// Add event listener for opening and closing details
$('#example tbody').on('click', 'td.details-control', function () {
$('#eccstable tbody').on('click', 'td.details-control', function () {
var tr = $(this).closest('tr');
var row = table.row( tr );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment