Commit 0dfb7554 authored by Cloud User's avatar Cloud User
Browse files

nitial commit

parent 743e40d0
# FTICKS-ELK-Ansible
# Ansible Playbook to deploy Federation FTICKS collector based on ELK stack
1. [The Playbook](#the-playbook)
2. [Contacts](#contacts)
## The Playbook
This playbook provides an easy way to deploy a SAML2 federation FTICKS collector node.
It will install and configure:
1. Elasticsearch cluster (role elastic.elasticsearch)
2. Logstash (role elastic.logstash)
3. Kibana + Nginx (role elastic.kibana)
A sample inventory is also available in the directory "inventory". It contans a group_vars directory.
## Deployment
To use this ansible playbook, one needs to define the inventory and fill in the information in the group_vars indicated with CHANGE ME.
To deploy the FTICKS collector, one needs to execute
ansible-playbook -i inventory multinode.yml
## Usage
The deployment will deploy a secure Elastic cluster. The Kibana is available on the designated node on port 443, where public readonly access is available.
On the Kibana node port 8443, there is a secure access to the Kibana.
## IDPs
The Federation IDPs need to send data on port 514 to the Logstash node IP address.
## NOTE
The Firewall is not configured using this ansible playbook. Only on logstash node a port redirection is done from port 514 to port 1514, since Logstash does not listen on privilege ports.
[defaults]
es_version: 7.6.2
es_api_basic_auth_username: elastic
# CHANGE ME
es_api_basic_auth_password: D23eV#F4d
# CHANGE ME to first node in Elastic cluster
es_api_config_node: <IP of node_0>
# If you do not want to use default location
#es_data_dirs:
# - "/opt/elasticsearch/data"
#es_log_dir: "/opt/elasticsearch/logs"
# Increase this
es_heap_size: "1g"
es_plugins: []
es_enable_xpack: true
es_xpack_features: []
es_xpack_trial: false
# Create your own keystore and CA once you install elasticsearch and put them in roles/elastic.elasticsearch/files/
#es_ssl_keystore: "my-keystore.p12"
#es_ssl_keystore_password: ""
#es_ssl_truststore: "my-ca.p12"
#es_ssl_truststore_password: ""
#es_validate_certs: "no"
# CHANGE to true after the first install and certs generated
es_enable_auto_ssl_configuration: false
es_enable_transport_ssl: false
kibana_version: "7.x"
kibana_server_host: "127.0.0.1"
# CHANGE ME
kibana_server_name: fticks.feredeation.local
# Restricted user for kibana
kibana_nginx_user: readonly_user
# CHANGE ME
kibana_nginx_pass: C23c3rn0324-0md3
# CHANGE ME
# Use 'echo -n "user:pass" | base64'
kibana_nginx_user_pass: cmVhZG9ubHlfdXNlcjpDMjNjM3JuMDMyNC0wbWQz
kibana_readonly_role: "readonly"
kibana_readonly_space: "readonly"
logstash_version: "7.x"
logstash_install_plugins:
- logstash-input-syslog
- logstash-output-syslog
# CHANGE ME
es_hosts: '"<node_0_IP>:9200","<node_1_IP>:9200","<node_2_IP>:9200"'
# CHANGE ME
es_config_hosts: <node_0_IP>
es_logstash_fticks_user: fticks
# CHANGE ME
es_logstash_fticks_password: D4d313fr
es_logstash_fticks_role: fticks
fticks_central_server: "collector.f-ticks.edugain.org"
all:
children:
elasticsearch:
hosts:
node_0:
# ansible_host: <IP>
node_1:
# ansible_host: <IP>
node_2:
# ansible_host: <IP>
# vars:
# ansible_python_interpreter: "/usr/bin/python3"
logstash:
hosts:
node_3:
# ansible_host: <IP>
kibana:
hosts:
node_4:
# ansible_host: <IP>
- hosts: node_0
roles:
- role: elastic.elasticsearch
tags:
- elasticsearch
- role: elastic.logstash
tags:
- logstash
vars:
es_config:
node.name: master_node
node.data: true
node.master: true
cluster.name: "test-cluster"
cluster.initial_master_nodes: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300"
discovery.seed_hosts: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300,194.149.135.37:9300"
http.host: 0.0.0.0
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: false
- hosts: node_0
roles:
- hosts: node_1
roles:
- role: elastic.elasticsearch
tags:
- elasticsearch
vars:
es_config:
node.name: data_node_1
node.data: true
node.master: true
cluster.name: "test-cluster"
cluster.initial_master_nodes: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300"
discovery.seed_hosts: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300,194.149.135.37:9300"
http.host: 0.0.0.0
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: false
- hosts: node_2
roles:
- role: elastic.elasticsearch
tags:
- elasticsearch
vars:
es_config:
node.name: data_node_2
node.data: true
node.master: true
cluster.name: "test-cluster"
cluster.initial_master_nodes: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300"
discovery.seed_hosts: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300,194.149.135.37:9300"
http.host: 0.0.0.0
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: false
- hosts: node_3
roles:
- role: elastic.elasticsearch
tags:
- elasticsearch
- role: elastic.kibana
tags:
- kibana
vars:
es_config:
node.name: data_node_3
node.data: false
node.master: false
cluster.name: "test-cluster"
cluster.initial_master_nodes: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300"
discovery.seed_hosts: "194.149.135.61:9300,194.149.135.107:9300,194.149.135.120:9300,194.149.135.37:9300"
http.host: 0.0.0.0
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: false
---
##### GLOBAL METADATA
- meta:
cluster: devops-ci
##### JOB DEFAULTS
- job:
project-type: matrix
logrotate:
daysToKeep: 30
numToKeep: 100
parameters:
- string:
name: branch_specifier
default: master
description: the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;,
&lt;commitId&gt;, etc.)
properties:
- github:
url: https://github.com/elastic/ansible-elasticsearch/
- inject:
properties-content: HOME=$JENKINS_HOME
concurrent: true
node: master
scm:
- git:
name: origin
credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
reference-repo: /var/lib/jenkins/.git-references/ansible-elasticsearch.git
branches:
- ${branch_specifier}
url: git@github.com:elastic/ansible-elasticsearch.git
basedir: elasticsearch
wipe-workspace: 'False'
axes:
- axis:
type: slave
name: label
values:
- linux
- axis:
name: VERSION
filename: elasticsearch/test/matrix.yml
type: yaml
- axis:
name: OS
filename: elasticsearch/test/matrix.yml
type: yaml
- axis:
name: TEST_TYPE
filename: elasticsearch/test/matrix.yml
type: yaml
vault:
role_id: cff5d4e0-61bf-2497-645f-fcf019d10c13
wrappers:
- ansicolor
- timeout:
type: absolute
timeout: 360
fail: true
- timestamps
publishers:
- email:
recipients: infra-root+build@elastic.co
---
- job:
name: elastic+ansible-elasticsearch+master
display-name: elastic / ansible-elasticsearch - master
description: Master branch testing with test kitchen
triggers:
- timed: H H(02-04) * * *
builders:
- shell: |-
#!/usr/local/bin/runbld
set -euo pipefail
export RBENV_VERSION='2.5.7'
export PATH="$HOME/.rbenv/bin:$PATH"
eval "$(rbenv init -)"
rbenv local $RBENV_VERSION
export ES_XPACK_LICENSE_FILE="$(pwd)/license.json"
echo "Getting xpack_license from secrets service"
set +x
VAULT_TOKEN=$( curl -s -X POST -H "Content-Type: application/json" -L -d "{\"role_id\":\"$VAULT_ROLE_ID\",\"secret_id\":\"$VAULT_SECRET_ID\"}" $VAULT_ADDR/v1/auth/approle/login | jq -r '.auth.client_token' )
curl -s -L -H "X-Vault-Token:$VAULT_TOKEN" $VAULT_ADDR/v1/secret/devops-ci/ansible-elasticsearch/xpack_license | jq -r '.data.value' > ${ES_XPACK_LICENSE_FILE}
set -x
echo "Finished getting xpack_license from secrets service"
make setup
make verify VERSION=$VERSION PATTERN=$TEST_TYPE-$OS
---
- job:
name: elastic+ansible-elasticsearch+pull-request
display-name: elastic / ansible-elasticsearch - pull-request
description: Pull request testing with test kitchen
project-type: matrix
parameters: []
scm:
- git:
branches:
- $ghprbActualCommit
refspec: +refs/pull/*:refs/remotes/origin/pr/*
triggers:
- github-pull-request:
github-hooks: true
org-list:
- elastic
allow-whitelist-orgs-as-admins: true
cancel-builds-on-update: true
status-context: devops-ci
builders:
- shell: |-
#!/usr/local/bin/runbld
set -euo pipefail
export RBENV_VERSION='2.5.7'
export PATH="$HOME/.rbenv/bin:$PATH"
eval "$(rbenv init -)"
rbenv local $RBENV_VERSION
export ES_XPACK_LICENSE_FILE="$(pwd)/license.json"
echo "Getting xpack_license from secrets service"
set +x
VAULT_TOKEN=$( curl -s -X POST -H "Content-Type: application/json" -L -d "{\"role_id\":\"$VAULT_ROLE_ID\",\"secret_id\":\"$VAULT_SECRET_ID\"}" $VAULT_ADDR/v1/auth/approle/login | jq -r '.auth.client_token' )
curl -s -L -H "X-Vault-Token:$VAULT_TOKEN" $VAULT_ADDR/v1/secret/devops-ci/ansible-elasticsearch/xpack_license | jq -r '.data.value' > ${ES_XPACK_LICENSE_FILE}
set -x
echo "Finished getting xpack_license from secrets service"
make setup
make verify VERSION=$VERSION PATTERN=$TEST_TYPE-$OS
<!--
** Please read the guidelines below. **
Issues that do not follow these guidelines are likely to be closed.
1. GitHub is reserved for bug reports and feature requests. The best place to
ask a general question is at the Elastic [forums](https://discuss.elastic.co).
GitHub is not the place for general questions.
2. Is this bug report or feature request for a supported OS? If not, it
is likely to be closed. See https://www.elastic.co/support/matrix#show_os
3. Please fill out EITHER the feature request block or the bug report block
below, and delete the other block.
-->
<!-- Feature request -->
**Describe the feature**:
<!-- Bug report -->
**Elasticsearch version**
**Role version**: (If using master please specify github sha)
**JVM version** (`java -version`):
**OS version** (`uname -a` if on a Unix-like system):
**Description of the problem including expected versus actual behaviour**:
**Playbook**:
Please specify the full playbook used to reproduce this issue.
**Provide logs from Ansible**:
**ES Logs if relevant**:
---
# Number of days of inactivity before an issue becomes stale
daysUntilStale: 90
# Number of days of inactivity before an stale issue is closed
daysUntilClose: 30
# Label to use when marking an issue as stale
staleLabel: triage/stale
issues:
# Comment to post when marking an issue as stale.
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
# Comment to post when closing a stale issue.
closeComment: >
This issue has been automatically closed because it has not had recent
activity since being marked as stale.
pulls:
# Comment to post when marking a PR as stale.
markComment: >
This PR has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
To track this PR (even if closed), please open a corresponding issue if one
does not already exist.
# Comment to post when closing a stale PR.
closeComment: >
This PR has been automatically closed because it has not had recent
activity since being marked as stale.
Please reopen when work resumes.
.kitchen/
license*.json
*.pyc
.vendor
.bundle
Converging
TODO
.idea/
elasticsearch.iml
!/vars/RedHat.yml
---
driver:
name: docker
provisioner:
name: ansible_playbook
hosts: localhost
roles_path: ../
require_ansible_repo: true
require_ansible_omnibus: false
require_ansible_source: false
http_proxy: <%= ENV['HTTP_PROXY'] %>
https_proxy: <%= ENV['HTTPS_PROXY'] %>
no_proxy: localhost,127.0.0.1
ignore_extensions_from_root: [".git",".idea",".kitchen.yml"]
ignore_paths_from_root: [".git",".idea",".kitchen"]
<% if ENV['VERSION'] %>
attributes:
extra_vars:
es_major_version: "<%= ENV['VERSION'] %>"
<% if ENV['VERSION'] == '6.x' %>
es_version: '6.8.8'
<% end %>
<% end %>
transport:
max_ssh_sessions: 6
platforms:
- name: ubuntu-14.04
driver_config:
image: ubuntu:14.04
privileged: true
provision_command:
- apt-get update -q && apt-get install -y -q software-properties-common && add-apt-repository -y ppa:ansible/ansible && add-apt-repository -y ppa:openjdk-r/ppa
- apt-get update -q && apt-get -y -q install ansible openjdk-8-jre python-jmespath
- locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8
use_sudo: false
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
- name: ubuntu-16.04
driver_config:
image: ubuntu:16.04
privileged: true
provision_command:
- apt-get update -q && apt-get install -y -q iproute locales software-properties-common && add-apt-repository -y ppa:ansible/ansible
- apt-get update -q && apt-get install -y -q ansible python-jmespath
- locale-gen en_US.UTF-8 && localedef -i en_US -c -f UTF-8 en_US.UTF-8
use_sudo: false
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
run_command: "/sbin/init"
- name: ubuntu-18.04
driver_config:
image: ubuntu:18.04
privileged: true
provision_command:
- apt-get install -y -q ansible iproute2 python-jmespath
use_sudo: false
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
run_command: "/sbin/init"
- name: debian-8
driver_config:
image: debian:8
privileged: true
provision_command:
- apt-get update -q && apt-get install -y -q gnupg2 python-jmespath
- echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list
- echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf
- apt-get update && apt-get -y install -t jessie-backports openjdk-8-jre-headless
- echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
- apt-get update -q && apt-get install -y -q ansible
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
use_sudo: false
run_command: "/sbin/init"
- name: debian-9
driver_config:
image: debian:9
privileged: true
provision_command:
- apt-get update -q && apt-get install -y -q gnupg2 python-jmespath systemd-sysv
- echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
- apt-get update -q && apt-get install -y -q ansible
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
use_sudo: false
run_command: "/sbin/init"
- name: centos-7
driver_config:
image: centos:7
provision_command:
- yum -y install epel-release
- yum -y install ansible iproute python2-jmespath
volume:
- <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
- /etc # This fixes certain java file actions that check the mount point. Without this adding users fails for some docker storage drivers
run_command: "/usr/sbin/init"
privileged: true
use_sudo: false
suites:
- name: oss
provisioner:
idempotency_test: true
playbook: test/integration/oss.yml
- name: oss-upgrade
provisioner:
playbook: test/integration/oss-upgrade.yml
idempotency_test: false
- name: oss-to-xpack-upgrade
provisioner:
playbook: test/integration/oss-to-xpack-upgrade.yml
idempotency_test: false
- name: xpack
provisioner:
playbook: test/integration/xpack.yml
idempotency_test: true
- name: xpack-upgrade
provisioner:
playbook: test/integration/xpack-upgrade.yml
idempotency_test: false
- name: issue-test
provisioner:
playbook: test/integration/issue-test.yml
idempotency_test: false
- name: xpack-upgrade-trial
provisioner:
playbook: test/integration/xpack-upgrade-trial.yml
idempotency_test: false
This diff is collapsed.
source 'https://rubygems.org'
gem 'test-kitchen'
gem 'kitchen-docker'
gem 'kitchen-ansible'
gem 'net-ssh'
GEM
remote: https://rubygems.org/
specs:
bcrypt_pbkdf (1.0.1)
builder (3.2.4)
ed25519 (1.2.4)
equatable (0.5.0)
erubi (1.9.0)
ffi (1.12.1)
gssapi (1.3.0)
ffi (>= 1.0.1)
gyoku (1.3.1)
builder (>= 2.1.2)
httpclient (2.8.3)
kitchen-ansible (0.50.0)
net-ssh (>= 3)
test-kitchen (>= 1.4)
kitchen-docker (2.9.0)
test-kitchen (>= 1.0.0)
license-acceptance (1.0.11)