Неназначенные сегменты Elasticsearch CircuitBreakingException [[parent] Данные слишком велики

Я получил предупреждение о том, что у elasticsearch есть 2 неназначенных осколка. Я сделал ниже вызовы API, чтобы собрать более подробную информацию.

    curl -s http://localhost:9200/_cluster/allocation/explain | python -m json.tool

Результат ниже

    "allocate_explanation": "cannot allocate because allocation is not permitted to any of the nodes",
    "can_allocate": "no",
    "current_state": "unassigned",
    "index": "docs_0_1603929645264",
    "node_allocation_decisions": [
        {
            "deciders": [
                {
                    "decider": "max_retry",
                    "decision": "NO",
                    "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-10-30T06:10:16.305Z], failed_attempts[5], delayed=false, details[failed shard on node [o_9jyrmOSca9T12J4bY0Nw]: failed recovery, failure RecoveryFailedException[[docs_0_1603929645264][0]: Recovery failed from {elasticsearch-data-1}{fIaSuZsNTwODgZnt90f7kQ}{Qxl9iPacQVS-tN_t4YJqrw}{IP1}{IP:9300} into {elasticsearch-data-0}{o_9jyrmOSca9T12J4bY0Nw}{1w5mgwy0RYqBQ9c-qA_6Hw}{IP}{IP:9300}]; nested: RemoteTransportException[[elasticsearch-data-1][IP:9300][internal:index/shard/recovery/start_recovery]]; nested: RecoveryEngineException[Phase[1] phase1 failed]; nested: RecoverFilesRecoveryException[Failed to transfer [129] files with total size of [4.4gb]]; nested: RemoteTransportException[[elasticsearch-data-0][IP2:9300][internal:index/shard/recovery/file_chunk]]; nested: 
CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [1972835086/1.8gb], which is larger than the limit of [1972122419/1.8gb], real usage: [1972833976/1.8gb], new bytes reserved: [1110/1kb]]; ], allocation_status[no_attempt]]]"
                }
            ],
            "node_decision": "no",
            "node_id": "1XEXS92jTK-asdfasdfasdf",
            "node_name": "elasticsearch-data-2",
            "transport_address": "IP1:9300"
        },
        {
            "deciders": [
                {
                    "decider": "max_retry",
                    "decision": "NO",
                    "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-10-30T06:10:16.305Z], failed_attempts[5], delayed=false, details[failed shard on node [o_9jyrmOSca9T12J4bY0Nw]: failed recovery, failure RecoveryFailedException[[docs_0_1603929645264][0]: Recovery failed from {elasticsearch-data-1}{fIaSuZsNTwODgZnt90f7kQ}{Qxl9iPacQVS-tN_t4YJqrw}{IP1}{IP1:9300} into {elasticsearch-data-0}{o_9jyrmOSca9T12J4bY0Nw}{1w5mgwy0RYqBQ9c-qA_6Hw}{IP2}{IP2:9300}]; nested: RemoteTransportException[[elasticsearch-data-1][IP1:9300][internal:index/shard/recovery/start_recovery]]; nested: RecoveryEngineException[Phase[1] phase1 failed]; nested: RecoverFilesRecoveryException[Failed to transfer [129] files with total size of [4.4gb]]; nested: RemoteTransportException[[elasticsearch-data-0][IP2:9300][internal:index/shard/recovery/file_chunk]]; nested: 
CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [1972835086/1.8gb], which is larger than the limit of [1972122419/1.8gb], real usage: [1972833976/1.8gb], new bytes reserved: [1110/1kb]]; ], allocation_status[no_attempt]]]"
                },
                {
                    "decider": "same_shard",
                    "decision": "NO",
                    "explanation": "the shard cannot be allocated to the same node on which a copy of the shard already exists [[docs_0_1603929645264][0], node[fIaSuZsNTwODgZnt90f7kQ], [P], s[STARTED], a[id=stHnyqjLQ7OwFbaqs5vWqA]]"
                }
            ],
            "node_decision": "no",
            "node_id": "fIaSuZsNTwODgZnt90f7kQ",
            "node_name": "elasticsearch-data-1",
            "transport_address": "IP1:9300"
        },
        {
            "deciders": [
                {
                    "decider": "max_retry",
                    "decision": "NO",
                    "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2020-10-30T06:10:16.305Z], failed_attempts[5], delayed=false, details[failed shard on node [o_9jyrmOSca9T12J4bY0Nw]: failed recovery, failure RecoveryFailedException[[docs_0_1603929645264][0]: Recovery failed from {elasticsearch-data-1}{fIaSuZsNTwODgZnt90f7kQ}{Qxl9iPacQVS-tN_t4YJqrw}{IP1}{IP1:9300} into {elasticsearch-data-0}{o_9jyrmOSca9T12J4bY0Nw}{1w5mgwy0RYqBQ9c-qA_6Hw}{Ip2}{IP2:9300}]; nested: RemoteTransportException[[elasticsearch-data-1][IP1:9300][internal:index/shard/recovery/start_recovery]]; nested: RecoveryEngineException[Phase[1] phase1 failed]; nested: RecoverFilesRecoveryException[Failed to transfer [129] files with total size of [4.4gb]]; nested: RemoteTransportException[[elasticsearch-data-0][IP2:9300][internal:index/shard/recovery/file_chunk]]; nested: 
CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [1972835086/1.8gb], which is larger than the limit of [1972122419/1.8gb], real usage: [1972833976/1.8gb], new bytes reserved: [1110/1kb]]; ], allocation_status[no_attempt]]]"
                }
            ],
            "node_decision": "no",
            "node_id": "o_9jyrmOSca9T12J4bY0Nw",
            "node_name": "elasticsearch-data-0",
            "transport_address": "IP2:9300"
        }
    ],
    "primary": false,
    "shard": 0,
    "unassigned_info": {
        "at": "2020-10-30T06:10:16.305Z",
        "details": "failed shard on node [o_9jyrmOSca9T12J4bY0Nw]: failed recovery, failure RecoveryFailedException[[docs_0_1603929645264][0]: Recovery failed from {elasticsearch-data-1}{fIaSuZsNTwODgZnt90f7kQ}{Qxl9iPacQVS-tN_t4YJqrw}{IP1}{IP1:9300} into {elasticsearch-data-0}{o_9jyrmOSca9T12J4bY0Nw}{1w5mgwy0RYqBQ9c-qA_6Hw}{IP2}{IP2:9300}]; nested: RemoteTransportException[[elasticsearch-data-1][IP1:9300][internal:index/shard/recovery/start_recovery]]; nested: RecoveryEngineException[Phase[1] phase1 failed]; nested: RecoverFilesRecoveryException[Failed to transfer [129] files with total size of [4.4gb]]; nested: RemoteTransportException[[elasticsearch-data-0][IP2:9300][internal:index/shard/recovery/file_chunk]]; nested: 
CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [1972835086/1.8gb], which is larger than the limit of [1972122419/1.8gb], real usage: [1972833976/1.8gb], new bytes reserved: [1110/1kb]]; ",
        "failed_allocation_attempts": 5,
        "last_allocation_status": "no_attempt",
        "reason": "ALLOCATION_FAILED"
    }
}

Я запросил конфигурацию выключателя

    curl -X GET "localhost:9200/_nodes/stats/breaker?pretty

И вы можете видеть, что родительский limit_size_in_byes 3 узлов (elasticsearch-data-0, elasticsearch-data-1 и elasticsearch-data-2) выглядит следующим образом.

"parent" : {
          "limit_size_in_bytes" : 1972122419,
          "limit_size" : "1.8gb",
          "estimated_size_in_bytes" : 1648057776,
          "estimated_size" : "1.5gb",
          "overhead" : 1.0,
          "tripped" : 139
        }

Я сослался на этот ответ https://stackoverflow.com/a/61954408 и планирую увеличить либо процент памяти автоматического выключателя или общая куча JVM.

Это среда k8s, и данные elasticsearch развернуты как набор состояний с 3 репликами. Когда я описал набор состояний, я увидел, что указанная ниже переменная ENV определена

Containers:
   elasticsearch:
    Image:      custom/elasticsearch-oss-s3:7.0.0
    Port:       9300/TCP
    Host Port:  0/TCP
    Limits:
      cpu:     10500m
      memory:  21Gi
    Requests:
      cpu:      10
      memory:   20Gi
    Environment:
      DISCOVERY_SERVICE:     elasticsearch-discovery
      NODE_MASTER:           false
      PROCESSORS:            11 (limits.cpu)
      ES_JAVA_OPTS:          -Djava.net.preferIPv4Stack=true -Xms2048m -Xmx2048m

В соответствии с этим размер кучи составляет 2048 м.

Я вошел в модуль данных elasticsearch и в каталоге конфигурации эластичного поиска вижу следующие файлы

elasticsearch.keystore  elasticsearch.yml  jvm.options  log4j2.properties  repository-s3

elasticsearch.yml не имеет конфигурации кучи или как таковой. У него было просто имя главных узлов и т. Д.

Ниже находится файл jvm.options

## JVM configuration

# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space

-Xms1g
-Xmx1g


## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly


## DNS cache policy
-Des.networkaddress.cache.ttl=60
-Des.networkaddress.cache.negative.ttl=10


# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch

## basic

# explicitly set the stack size
-Xss1m

# set to headless, just in case
-Djava.awt.headless=true

# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8

# use our provided JNA always versus the system one
-Djna.nosys=true

# turn off a JDK optimization that throws away stack traces for common
# exceptions because stack traces are important for debugging
-XX:-OmitStackTraceInFastThrow


# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0

# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true

-Djava.io.tmpdir=${ES_TMPDIR}

## heap dumps

# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError

# specify an alternative path for heap dumps; ensure the directory exists and
# has sufficient space
-XX:HeapDumpPath=data

# specify an alternative path for JVM fatal error logs
-XX:ErrorFile=logs/hs_err_pid%p.log

## JDK 8 GC logging

8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m

# JDK 9+ GC logging
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
# time/date parsing will break in an incompatible way for some date patterns and locals
9-:-Djava.locale.providers=COMPAT

Сверху кажется, что общий размер кучи составляет 1 г.

Но из переменной env, определенной в наборе с отслеживанием состояния этого модуля, кажется, что это 2048 м.

Какой из них правильный?

Теперь по ссылке ниже

Настройки автоматического выключателя | Elasticsearch

Прерыватель родительского уровня может быть настроен со следующими настройками:

index.breaker.total.use_real_memory (статический) Определяет, должен ли родительский выключатель учитывать реальное использование памяти (true) или учитывать только количество, зарезервированное дочерними автоматическими выключателями (false). По умолчанию true.

index.breaker.total.limit (динамический) Предел запуска для общего родительского выключателя. По умолчанию 70% кучи JVM, если index.breaker.total.use_real_memory имеет значение false. Если index.breaker.total.use_real_memory имеет значение true, по умолчанию используется 95% кучи JVM.

Но предельное значение ошибки и запрошенной мной статистики выключателя - 1972122419 байт (1,8 ГБ). Это не похоже на 95% от 2048M или 1g.

Теперь, как я могу увеличить кучу или предел памяти родительского прерывателя, чтобы избавиться от этой ошибки?