Commit 846b364e authored by Marco Malavolti's avatar Marco Malavolti
Browse files

Add properties and other refactor

parent 3eadc8b9
......@@ -6,7 +6,9 @@
* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv)
* `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
* `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
* `cd eccs2 ; ./eccs2.py`
* `cd eccs2`
* `cp eccs2properties.py.template eccs2properties.py` (and change it with your needs)
* `./runEccs2.py`
# API Development Server
......
......@@ -4,18 +4,20 @@ import argparse
import json
import logging
import os
import eccs2properties
import signal
import re
import requests
from datetime import date
from eccs2properties import ECCS2LOGSPATH, ECCS2RESULTSLOG, ECCS2CHECKSLOG, ECCS2SELENIUMLOG, FEDS_BLACKLIST, IDPS_BLACKLIST
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from urllib3.exceptions import MaxRetryError
"""
......@@ -27,8 +29,8 @@ from selenium.common.exceptions import TimeoutException
def checkIdP(sp,idp,logger,driver):
# Configure Blacklists
federation_blacklist = eccs2properties.FEDS_BLACKLIST
entities_blacklist = eccs2properties.IDPS_BLACKLIST
federation_blacklist = FEDS_BLACKLIST
entities_blacklist = IDPS_BLACKLIST
if (idp['registrationAuthority'] in federation_blacklist):
logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
......@@ -46,20 +48,46 @@ def checkIdP(sp,idp,logger,driver):
except TimeoutException as e:
logger.info("%s;%s;999;TIMEOUT" % (idp['entityID'],sp))
return "TIMEOUT"
except WebDriverException as e:
print("!!! WEB DRIVER EXCEPTION !!!")
raise e
except Exception as e:
print ("!!! EXCEPTION !!!")
raise e
"""
except MaxRetryError as e:
logger.info("%s;%s;111;MaxRetryError" % (idp['entityID'],sp))
return "MaxRetryError"
except ConnectionRefusedError as e:
logger.info("%s;%s;222;ConnectionRefusedError" % (idp['entityID'],sp))
return "ConnectionRefusedError"
except ConnectionError as e:
logger.info("%s;%s;333;ConnectionError" % (idp['entityID'],sp))
return "ConnectionError"
except NoSuchElementException as e:
print("!!! NO SUCH ELEMENT EXCEPTION !!!")
print(e.__str__())
pass
except WebDriverException as e:
"""
"""
if "ConnectionRefusedError" in e.__str__():
logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
return "Connection-Error"
return "ConnectionRefusedError"
elif "Connection Refused" in e.__str__():
logger.info("%s;%s;000;ConnectionRefused" % (idp['entityID'],sp))
return "Connection-Refused"
else:
print("!!! UN-HANDLE WEB DRIVER EXCEPTION !!!")
raise e
except:
print("!!! UN-HANDLE OTHER EXCEPTION !!!")
raise e
"""
pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
......@@ -75,16 +103,9 @@ def checkIdP(sp,idp,logger,driver):
status_code = r.status_code
except requests.exceptions.ConnectionError as e:
driver.delete_all_cookies()
driver.close()
driver.quit()
logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
return "Connection-Error"
except requests.exceptions.RequestException as e:
driver.delete_all_cookies()
driver.close()
driver.quit()
print("!!! UN-HANDLE REQUEST EXCEPTION !!!")
raise SystemExit(e)
......@@ -103,10 +124,10 @@ def checkIdP(sp,idp,logger,driver):
# Use logger to produce files consumed by ECCS-2 API
def getLogger(filename,log_level="DEBUG",path="./"):
def getLogger(filename, path=".", log_level="DEBUG"):
logger = logging.getLogger(filename)
ch = logging.FileHandler(path+filename,'a','utf-8')
ch = logging.FileHandler(path + '/' + filename,'a','utf-8')
if (log_level == "DEBUG"):
logger.setLevel(logging.DEBUG)
......@@ -138,7 +159,10 @@ def getIdPContacts(idp,contactType):
for ctcType in idp['contacts']:
if (ctcType == contactType):
for ctc in idp['contacts'][contactType]:
ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
if (ctc['emailOrPhone'].get('EmailAddress')):
ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
else:
ctcList.append('missing')
return ctcList
......@@ -196,8 +220,8 @@ def checkIdp(idp,sps,eccs2log,eccs2checksLog,driver):
# MAIN
if __name__=="__main__":
eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSPATH, "INFO")
eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSPATH, "INFO")
sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
......@@ -220,9 +244,9 @@ if __name__=="__main__":
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--disable-extensions')
#driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=./selenium_chromedriver.log'])
driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=%s' % ECCS2SELENIUMLOG])
#driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
driver = webdriver.Chrome('chromedriver', options=chrome_options)
#driver = webdriver.Chrome('chromedriver', options=chrome_options)
# Configure timeouts: 30 sec
driver.set_page_load_timeout(30)
......
#!/usr/bin/env python3.8
from datetime import date
day = date.today().isoformat()
ECCS2PATH = "/opt/eccs2"
ECCS2LOGSPATH = "%s/logs" % ECCS2PATH
ECCS2RESULTSLOG = "eccs2_%s.log" % day
ECCS2CHECKSLOG = "eccs2checks_%s.log" % day
ECCS2SELENIUMLOG = "%s/selenium_chromedriver.log" % ECCS2LOGSPATH
ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSPATH
ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSPATH
ECCS2NUMPROCESSES = 20
FEDS_BLACKLIST = [
'<REGISTRATION-AUTHORITY-1>',
'<REGISTRATION-AUTHORITY-2>',
'<REGISTRATION-AUTHORITY-3>',
]
IDPS_BLACKLIST = [
'<ENTITYID-IDP-1>',
'<ENTITYID-IDP-2>',
'<ENTITYID-IDP-3>',
]
......@@ -8,9 +8,10 @@ import requests
import sys
import time
from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2PATH, ECCS2NUMPROCESSES
from subprocess import Popen,PIPE
# returns a Dict on "{ nameFed:reg_auth }"
# Returns a Dict on "{ nameFed:reg_auth }"
def getRegAuthDict(list_feds):
regAuth_dict = {}
......@@ -23,7 +24,7 @@ def getRegAuthDict(list_feds):
return regAuth_dict
# returns a list of IdP for a single federation
# Returns a list of IdP for a single federation
def getIdpList(list_eccs_idps,reg_auth):
fed_idp_list = []
......@@ -44,7 +45,7 @@ def getListFeds(url, filename):
# then open it and work with local file
with open("%s" % (filename), mode="r", encoding='utf-8') as f:
return json.loads(f.read())
return json.loads(f.read().replace("'", "&apos;"))
# Returns a Python List
......@@ -57,16 +58,9 @@ def getListEccsIdps(url, filename):
# then open it and work with local file
with open("%s" % (filename), mode="r", encoding='utf-8') as f:
return json.loads(f.read())
# Prepare input file for ECCS2
def genEccs2input(reg_auth_dict):
for name,regAuth in reg_auth_dict.items():
fed_idp_list = getIdpList(list_eccs_idps,regAuth)
filename = "/tmp/data/inputEccs2/%s.txt" % name
with open("%s" % (filename), mode="w+", encoding='utf-8') as f:
f.write(','.join(str(idp) for idp in fed_idp_list))
return json.loads(f.read().replace("'", "&apos;"))
# Run Command
async def run(name,queue,stdout_file,stderr_file):
while True:
# Get a "cmd item" out of the queue.
......@@ -82,9 +76,9 @@ async def run(name,queue,stdout_file,stderr_file):
stdout, stderr = await proc.communicate()
if stdout:
stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n[stdout]\n{stdout.decode()}')
stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
if stderr:
stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n[stderr]\n{stderr.decode()}')
stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
# Notify the queue that the "work cmd" has been processed.
queue.task_done()
......@@ -102,7 +96,7 @@ async def main(cmd_list,stdout_file,stderr_file):
tasks = []
#for i in range(15): # !!!-WORKING-!!!
#for i in range(30): # !!!-WORSTE-!!!
for i in range(10):
for i in range(ECCS2NUMPROCESSES):
task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
tasks.append(task)
......@@ -134,18 +128,17 @@ if __name__=="__main__":
filename = "/tmp/data/list_eccs_idps.txt"
list_eccs_idps = getListEccsIdps(url, filename)
stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
stdout_file = open(ECCS2STDOUT,"w+")
stderr_file = open(ECCS2STDERR,"w+")
# Prepare input file for ECCS2
regAuthDict = getRegAuthDict(list_feds)
#genEccs2input(regAuthDict)
for name,regAuth in regAuthDict.items():
idpJsonList = getIdpList(list_eccs_idps,regAuth)
num_idps = len(idpJsonList)
cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
proc_list = []
count = 0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment