Set up VMware Aria Operations configuration for third-party Prometheus exporters

The steps to set up the configuration file for the first time are as follows:

1. 點擊左側選單 -> Operations -> Configuration -> Management packs Configuration

2. 點擊 ADD

3. 填入以下內容：

   • Name: k8s-custom-prom-metric-mappings.xml 注意，設定檔的名字不能更改，只能設定這名字
   • 設定值如下：

     <MappingDefinitions
         xmlns:xs="http://www.w3.org/2001/XMLSchema">
         <CustomMetricConfig collectCustomMetrics="true"
             enableAllLabels="false"
             labelNamesToExclude="beta_kubernetes_io_os,container_label_annotation_io_kubernetes_container_hash,container_label_annotation_io_kubernetes_container_ports,container_label_annotation_io_kubernetes_container_restartCount,container_label_annotation_io_kubernetes_container_terminationMessagePath,container_label_annotation_io_kubernetes_container_terminationMessagePolicy,container_label_annotation_io_kubernetes_pod_terminationGracePeriod,container_label_annotation_kubeadm_kubernetes_io_etcd_advertise_client_urls,container_label_annotation_kubeadm_kubernetes_io_kube_apiserver_advertise_address_endpoint,container_label_annotation_kubernetes_io_config_hash,container_label_annotation_kubernetes_io_config_seen,container_label_annotation_kubernetes_io_config_source,container_label_app,container_label_app_kubernetes_io_name,container_label_app_kubernetes_io_version,container_label_component,container_label_controller_revision_hash,container_label_description,container_label_io_kubernetes_container_logpath,container_label_io_kubernetes_docker_type,container_label_io_kubernetes_pod_uid,container_label_io_kubernetes_sandbox_id,container_label_k8s_app,container_label_maintainer,container_label_maintainers,container_label_pod_template_generation,container_label_pod_template_hash,container_label_purpose,container_label_tier,container_label_version,container_runtime_version,dockerVersion,goversion,image,image_id,instance,timezone,container_label_io_kubernetes_container_name,container_label_io_kubernetes_pod_name,container_label_io_kubernetes_pod_namespace,name"
             maxGroupNames="3"></CustomMetricConfig>
         <IdentifierMappings>
             <NodeIdentifierMapping
                 labelContainingNodeName="nodename,node_name,node,kubernetes_node_name" />
             <NamespaceIdentifierMapping
                 labelContainingNamespaceName="namespace,kubernetes_namespace,container_label_io_kubernetes_pod_namespace,source_workload_namespace" />
             <PodIdentifierMapping
                 labelContainingPodName="pod,pod_name,podname,container_label_io_kubernetes_pod_name" />
             <ContainerIdentifierMapping
                 labelContaingContainerID="id,container_id"
                 labelContainingContainerName="container_name,container" />
             <DeploymentIdentifierMapping
                 labelContainingDeploymentName="deployment" />
             <ReplicasetIdentifierMapping
                 labelContainingReplicasetName="replicaset" />
         </IdentifierMappings>
         <MappingDefinition resourceKind="namespace">
             <MetricGroup groupName="CXS-PVC-Prometheus" isInstanced="true">
                 <Metric instancedLabelName="persistentvolumeclaim"
                         metricName="Persistent Volume Claim Size in Bytes"
                         metricUnit="Bytes"
                         isRate="false"
                         rule='kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="$NAMESPACE_NAME"}'
                         ruleName='persistentvolumeclaim'/>
             </MetricGroup>
             <MetricGroup groupName="CXS-ETCD-Prometheus" isInstanced="true">
                 <Metric instancedLabelName="operation,resource"
                         metricName="Etcd Request Duration Seconds Sum"
                         metricUnit=""
                         isRate="false"
                         rule='etcd_request_duration_seconds_sum'
                         ruleName='etcd_request_duration_seconds_sum'/>
                 <Metric instancedLabelName="operation,resource"
                         metricName="Etcd Request Duration Seconds Count"
                         metricUnit=""
                         isRate="false"
                         rule='etcd_request_duration_seconds_count'
                         ruleName='etcd_request_duration_seconds_count'/>
                 <Metric instancedLabelName="operation,resource"
                         metricName="ETCD Average Per Request Latency"
                         metricUnit="sec"
                         isRate="true"
                         rule='rate(etcd_request_duration_seconds_sum[5m]) / rate(etcd_request_duration_seconds_count[5m])'
                         ruleName='etcd_average_request_latency'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD Average Request Latency"
                         metricUnit="sec"
                         isRate="true"
                         rule='sum(rate(etcd_request_duration_seconds_sum[5m])) / sum(rate(etcd_request_duration_seconds_count[5m]))'
                         ruleName='etcd_average_request_latency'/>
                 <Metric instancedLabelName="operation,resource"
                         metricName="Etcd Request Duration Seconds bucket"
                         metricUnit=""
                         isRate="false"
                         rule='etcd_request_duration_seconds_bucket'
                         ruleName='etcd_request_duration_seconds_bucket'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD P95 Request Duration"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.95, sum by (le) (rate(etcd_request_duration_seconds_bucket[5m])))'
                         ruleName='etcd_p95_request_duration'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD P99 Request Duration"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.99, sum by (le) (rate(etcd_request_duration_seconds_bucket[5m])))'
                         ruleName='etcd_p99_request_duration'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD Request Error Rate (%)"
                         metricUnit=""
                         isRate="true"
                         rule='( sum(rate(etcd_request_errors_total[5m])) / sum(rate(etcd_requests_total[5m])) ) * 100'
                         ruleName='etcd_request_error_rate'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD Request Rate (RPS)"
                         metricUnit=""
                         isRate="false"
                         rule='sum(rate(etcd_requests_total[1m]))'
                         ruleName='etcd_request_rate'/>
             </MetricGroup>
             <MetricGroup groupName="CXS-API-Server-Prometheus" isInstanced="true">
                 <Metric instancedLabelName=""
                         metricName="API Server Request Execution Duration (p95)"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.95, sum by (le) (rate(apiserver_flowcontrol_request_execution_seconds_bucket[5m])))'
                         ruleName='api_server_request_execution_duration_p95'/>
                 <Metric instancedLabelName=""
                         metricName="API Server Request Execution Duration (p99)"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.99, sum by (le) (rate(apiserver_flowcontrol_request_execution_seconds_bucket[5m])))'
                         ruleName='api_server_request_execution_duration_p99'/>
                 <Metric instancedLabelName=""
                         metricName="API Server Request Wait Duration (p95)"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.95, sum by (le) (rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket[5m])))'
                         ruleName='api_server_request_wait_duration_p95'/>
                 <Metric instancedLabelName=""
                         metricName="API Server Request Wait Duration (p99)"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.99, sum by (le) (rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket[5m])))'
                         ruleName='api_server_request_wait_duration_p99'/>
                 <Metric instancedLabelName=""
                         metricName="API Server 5xx Error Rate (%)"
                         metricUnit=""
                         isRate="false"
                         rule='(sum(rate(apiserver_flowcontrol_dispatched_requests_total{code=~"5.."}[5m])) or vector(0)) / sum(rate(apiserver_flowcontrol_dispatched_requests_total[5m])) * 100'
                         ruleName='api_server_5xx_error_rate'/>
                 <Metric instancedLabelName=""
                         metricName="API Server In-flight Requests"
                         metricUnit=""
                         isRate="false"
                         rule='100 * sum(apiserver_current_inflight_requests) / clamp_min(max_over_time((sum(apiserver_current_inflight_requests))[7d:5m]), 1)'
                         ruleName='api_server_in_flight_requests'/>
                 <Metric instancedLabelName=""
                         metricName="API Server Long-running Requests"
                         metricUnit=""
                         isRate="false"
                         rule='100 * sum(apiserver_longrunning_requests) / avg_over_time((sum(apiserver_longrunning_requests))[7d:5m])'
                         ruleName='api_server_long_running_requests'/>
             </MetricGroup>
             <MetricGroup groupName="CXS-CoreDNS-Prometheus" isInstanced="true">
                 <Metric instancedLabelName=""
                         metricName="CoreDNS DNS Query P95 Latency"
                         metricUnit=""
                         isRate="false"
                         rule='histogram_quantile(0.95, sum(rate(coredns_dns_request_duration_seconds_bucket[5m])) by (le))'
                         ruleName='coredns-dns-query-p95-latency'/>
                 <Metric instancedLabelName=""
                         metricName="CoreDNS DNS Query Rate"
                         metricUnit=""
                         isRate="false"
                         rule='sum(rate(coredns_dns_requests_total[1m]))'
                         ruleName='coredns-dns-query-rate'/>
             </MetricGroup>
         </MappingDefinition>
         <MappingDefinition resourceKind="container">
             <MetricGroup groupName="CXS-Prometheus" isInstanced="true">
                 <Metric instancedLabelName=""
                         metricName="Status Running"
                         metricUnit=""
                         isRate="false"
                         rule='kube_pod_container_status_running'
                         ruleName='kube_pod_container_status_running'/>
                 <Metric instancedLabelName=""
                         metricName="Pod Restart Count"
                         metricUnit=""
                         isRate="false"
                         rule='increase(kube_pod_container_status_restarts_total[1h])'
                         ruleName='pod-restart-count'/>
                 <Metric instancedLabelName=""
                         metricName="Container Cpu Usage Seconds Total[5m]"
                         metricUnit="sec"
                         isRate="true"
                         rule='sum(rate(container_cpu_usage_seconds_total{namespace="$NAMESPACE_NAME",container="$CONTAINER_NAME"}[5m])) * 100'
                         ruleName='container_cpu_usage_seconds_total[5m]'/>
             </MetricGroup>
         </MappingDefinition>
         <MappingDefinition resourceKind="node">
             <MetricGroup groupName="CXS-filesystem" isInstanced="true">
                 <Metric instancedLabelName="mountpoint"
                         metricName="Filesystem Avail Space"
                         metricUnit="Bytes" promMetricName="node_filesystem_avail_bytes" isRate="false"/>
                 <Metric instancedLabelName="mountpoint"
                         metricName="Filesystem Size"
                         metricUnit="Bytes" promMetricName="node_filesystem_size_bytes" isRate="false"/>
             </MetricGroup>
             <MetricGroup groupName="CXS-ETCD_metrics" isInstanced="true">
                 <Metric instancedLabelName=""
                         metricName="ETCD Database Used Size"
                         metricUnit="Bytes"
                         isRate="false"
                         rule='etcd_mvcc_db_total_size_in_bytes'
                         ruleName='etcd_mvcc_db_total_size_in_bytes'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD Database Backend Size"
                         metricUnit="Bytes"
                         isRate="false"
                         rule='etcd_server_quota_backend_bytes'
                         ruleName='etcd_server_quota_backend_bytes'/>
                 <Metric instancedLabelName=""
                         metricName="ETCD Database Usage(%)"
                         metricUnit=""
                         isRate="false"
                         rule='(etcd_mvcc_db_total_size_in_bytes / etcd_server_quota_backend_bytes) * 100'
                         ruleName='etcd_server_db_usage'/>
             </MetricGroup>
         </MappingDefinition>
     </MappingDefinitions>
   • 點擊左下角 SAVE 按鈕

4. 讓 operations 重新套用設定檔 (1)

   

5. 讓 operations 重新套用設定檔 (2)

6. 看 log

7. 確認有收到 Metrics

參考文件

• Set up VMware Aria Operations configuration for third-party Prometheus exporters - Broadcom Docs