diff --git a/group_vars/default/pvc.yml b/group_vars/default/pvc.yml index a2ba452..5f08cb7 100644 --- a/group_vars/default/pvc.yml +++ b/group_vars/default/pvc.yml @@ -146,45 +146,6 @@ pvc_sriov_enable: False # Lowering the stack limit may cause poor performance or crashes in Zookeeper during some tasks. #pvc_zookeeper_stack_limit: 256M # 1/4 of default -# CPU pinning configuration via cset -# > ADVANCED TUNING: For most users, this is unnecessary and PVC will run fine with the default scheduling. -# Uncomment these options only for testing or if you are certain you meet the following conditions. -# > These options will tune cpuset (installed by default) to limit Ceph OSDs to certain CPU cores, while -# simultaneously limiting other system tasks and VMs to the remaining CPU cores. In effect it dedicates the -# specified CPU cores to Ceph OSDs only to ensure those processes can have dedicated CPU time. -# > Generally speaking, except in cases where extremely high random read throughput is required and in which -# the node(s) have a very large number of physical cores, this setting will not improve performance, and -# may in fact hurt performance. For more details please see the documentation. -# > For optimal performance when using this setting, you should dedicate exactly 2 cores, and their -# respective SMT threads if applicable, to each OSD. For instance, with 2 OSDs, 4 real cores (and their -# corresponding SMT threads if applicable) should be specified. More cores has been seen to, in some cases -# drop performance further. For more details please see the documentation. -# > Use the 'virsh capabilities' command to confim the exact CPU IDs (and SMT "siblings") for these lists. -# -pvc_shield_osds_enable: False -#pvc_shield_osds_cset: -# # This example host has 2x 6-core SMT-enabled CPUs; we want to use cores 0 (+SMT 12) and 2 (+SMT 14), which are -# # both on physical CPU 0, for 1x OSD. -# - hostname: pvchv1 -# osd_cset: -# - 0 -# - 2 -# - 12 -# - 14 -# # These example hosts have 1x 8-core SMT-enabled CPUs; we want to use cores 0 (+SMT 8) and 1 (+SMT 9) for 1x OSD. -# - hostname: pvchv2 -# osd_cset: -# - 0 -# - 1 -# - 8 -# - 9 -# - hostname: pvchv3 -# osd_cset: -# - 0 -# - 1 -# - 8 -# - 9 - # Configuration file networks # > Taken from base.yml's configuration; DO NOT MODIFY THIS SECTION. pvc_upstream_device: "{{ networks['upstream']['device'] }}" diff --git a/roles/pvc/tasks/ceph/main.yml b/roles/pvc/tasks/ceph/main.yml index 78c2532..d8773d8 100644 --- a/roles/pvc/tasks/ceph/main.yml +++ b/roles/pvc/tasks/ceph/main.yml @@ -138,56 +138,4 @@ command: ceph osd crush rule create-replicated replicated_rule default osd when: "{{ pvc_nodes | length }} == 1" -# System OSD CPU shielding activation -- block: - - name: install packages - apt: - name: - - cpuset - - numactl - state: latest - - - name: install ceph-osd-cpuset controller config - template: - src: ceph/ceph-osd-cpuset-enable.j2 - dest: /etc/default/ceph-osd-cpuset - - - name: install ceph-osd-cpuset script - template: - src: ceph/ceph-osd-cpuset.j2 - dest: /usr/local/sbin/ceph-osd-cpuset - mode: 0755 - - - name: install ceph-osd-cpuset service unit - template: - src: ceph/ceph-osd-cpuset.service.j2 - dest: /etc/systemd/system/ceph-osd-cpuset.service - register: systemd_file_cpuset - - - name: create ceph-osd override dropin directory - file: - dest: /etc/systemd/system/ceph-osd@.service.d - state: directory - - - name: install ceph-osd override dropin - template: - src: ceph/ceph-osd-cpuset.conf.j2 - dest: /etc/systemd/system/ceph-osd@.service.d/cpuset.conf - register: systemd_file_osd - - - name: reload systemd to apply previous changes - command: "systemctl daemon-reload" - when: systemd_file_cpuset.changed or systemd_file_osd.changed - - - name: enable ceph-osd-cpuset service - service: - name: ceph-osd-cpuset - enabled: yes - - - debug: - msg: "NOTICE: Any cpuset configs have NOT been applied to the running system. This node must be rebooted to apply these changes." - tags: pvc-ceph-cpuset - when: - - pvc_shield_osds_enable is defined - - meta: flush_handlers diff --git a/roles/pvc/templates/ceph/ceph-osd-cpuset-enable.j2 b/roles/pvc/templates/ceph/ceph-osd-cpuset-enable.j2 deleted file mode 100644 index 018c1ad..0000000 --- a/roles/pvc/templates/ceph/ceph-osd-cpuset-enable.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ pvc_shield_osds_enable }} diff --git a/roles/pvc/templates/ceph/ceph-osd-cpuset.conf.j2 b/roles/pvc/templates/ceph/ceph-osd-cpuset.conf.j2 deleted file mode 100644 index 8e1b7da..0000000 --- a/roles/pvc/templates/ceph/ceph-osd-cpuset.conf.j2 +++ /dev/null @@ -1,9 +0,0 @@ -# ceph-osd@.service overrides for cpuset -# {{ ansible_managed }} -[Service] -# Empty the existing ExecStart value -ExecStart = -# Set ExecStart to launch ceph-osd via cset proc inside the osd CSET -ExecStart = /usr/bin/cset proc --set=osd --exec /usr/bin/ceph-osd -- -f --cluster ${CLUSTER} --id %i --setuser ceph --setgroup ceph -# Disable control group protection, as cset uses control groups -ProtectControlGroups = false diff --git a/roles/pvc/templates/ceph/ceph-osd-cpuset.j2 b/roles/pvc/templates/ceph/ceph-osd-cpuset.j2 deleted file mode 100755 index 5d50079..0000000 --- a/roles/pvc/templates/ceph/ceph-osd-cpuset.j2 +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# PVC Ceph OSD cpuset preparation script -# {{ ansible_managed }} - -# This script is designed to prepare the cpusets for use by Ceph OSDs, VMs, and other system resources. -# Libvirt does not make this easy with any way to globally set its CPUs, so we must do this trickery. -{% if pvc_shield_osds_cset is defined %} -{% set cset_host = pvc_shield_osds_cset | selectattr('hostname', 'equalto', inventory_hostname) %} - -A_OSD_CPUS=( {{ cset_host[0]['osd_cset'] | join(' ') }} ) -A_SYS_CPUS=() -{% else %} - -A_OSD_CPUS=() -A_SYS_CPUS=() -{% endif %} - -CPU_INFO="$( lscpu )" - -# First, we must determine how many NUMA nodes we have -NUMA_COUNT="$( grep '^NUMA node(s)' <<<"${CPU_INFO}" | awk '{ print $NF }' )" - -# If we have 1 NUMA node, our SYS_MEMS is 0; otherwise it's 0-X -# This is needed to explicitly set our memspec during the set -if [[ ${NUMA_COUNT} -eq 1 ]]; then - SYS_MEMS="0" -else - SYS_MEMS="0-$(( ${NUMA_COUNT} - 1 ))" -fi - -# We must determine which NUMA nodes our OSD CPUS are in for the memspec during the set -A_OSD_MEMS=() -for CPU in ${A_OSD_CPUS[@]}; do - NODE="$( grep -E '^NUMA node[0-9]+ CPU' <<<"${CPU_INFO}" | grep -w "${CPU}" | awk '{ print $2 }' | sed 's/node//' )" - if [[ ! " ${A_OSD_MEMS} " =~ " ${NODE} " ]]; then - A_OSD_MEMS+=( $NODE ) - fi -done - -# Determine our CPU count -CPU_COUNT="$( grep '^CPU(s)' <<<"${CPU_INFO}" | awk '{ print $NF }' )" - -# Loop through all the CPUs in the count; if they are not in OSD_CPUS, add them to the SYS_CPUS array -for i in $( seq 0 $(( ${CPU_COUNT} - 1 )) ); do - if [[ ! " ${A_OSD_CPUS[*]} " =~ " ${i} " ]]; then - A_SYS_CPUS+=( $i ) - fi -done - -{% raw %} -if [[ $( cat /etc/default/ceph-osd-cpuset ) == "True" && ${#A_OSD_CPUS[@]} -gt 0 ]]; then -{% endraw %} - # Convert arrays into CSV - OSD_MEMS="$( IFS=, ; echo "${A_OSD_MEMS[*]}" )" - OSD_CPUS="$( IFS=, ; echo "${A_OSD_CPUS[*]}" )" - SYS_CPUS="$( IFS=, ; echo "${A_SYS_CPUS[*]}" )" -else - # Configs installed but disabled, so use all CPUs for everything - OSD_MEMS="${SYS_MEMS}" - OSD_CPUS="0-$(( ${CPU_COUNT} - 1 ))" - SYS_CPUS="0-$(( ${CPU_COUNT} - 1 ))" -fi - -echo "Enabled: $( cat /etc/default/ceph-osd-cpuset )" -echo "CPU count: ${CPU_COUNT}" -echo "OSD CPUs: ${OSD_CPUS}" -echo "OSD Mems: ${OSD_MEMS}" -echo "System/VM CPUs: ${SYS_CPUS}" -echo "System/VM Mems: ${SYS_MEMS}" - -# Create the system cpuset and move everything currently running into it -/usr/bin/cset set --cpu=${SYS_CPUS} --mem=${SYS_MEMS} system -/usr/bin/cset proc --move --force --threads root --toset=system - -# Create our Libvirt cpuset (identical to system cpuset) -/usr/bin/cset set --cpu=${SYS_CPUS} --mem=${SYS_MEMS} machine - -# Create our OSD cpuset -/usr/bin/cset set --cpu=${OSD_CPUS} --mem=${OSD_MEMS} osd diff --git a/roles/pvc/templates/ceph/ceph-osd-cpuset.service.j2 b/roles/pvc/templates/ceph/ceph-osd-cpuset.service.j2 deleted file mode 100644 index 6fa8f8c..0000000 --- a/roles/pvc/templates/ceph/ceph-osd-cpuset.service.j2 +++ /dev/null @@ -1,12 +0,0 @@ -# PVC Ceph OSD cpuset service unit -# {{ ansible_managed }} -[Unit] -Description = Ceph OSD cpuset shield creation -Before = ceph-osd@.service libvirtd.service - -[Service] -Type = oneshot -ExecStart = /usr/local/sbin/ceph-osd-cpuset - -[Install] -WantedBy = ceph.target