PUT _xpack/watcher/watch/cluster_health
{
  "metadata": {
    "not_green_secs": 60,
    "monitoring_update_interval": 10
  },
  "trigger": {
    "schedule": {
      "interval": "60s"
    }
  },
  "throttle_period": "30m",
  "input": {
    "search": {
      "request": {
        "search_type": "query_then_fetch",
        "indices": [
          ".monitoring-es*"
        ],
        "types": [
          "doc"
        ],
        "body": {
          "query": {
            "bool": {
              "filter": {
                "bool": {
                  "must": [
                    {
                      "range": {
                        "timestamp": {
                          "gte": "now-{{ctx.metadata.not_green_secs}}s"
                        }
                      }
                    },
                    {
                      "term": {
                        "type": {
                          "value": "cluster_stats"
                        }
                      }
                    }
                  ]
                }
              }
            }
          },
          "aggs": {
            "clusters": {
              "terms": {
                "field": "cluster_uuid",
                "size": 100
              },
              "aggs": {
                "cluster_state": {
                  "filters": {
                    "filters": {
                      "yellow": {
                        "term": {
                          "cluster_state.status": "yellow"
                        }
                      },
                      "red": {
                        "term": {
                          "cluster_state.status": "red"
                        }
                      }
                    }
                  }
                },
                "latest_state": {
                  "top_hits": {
                    "size": 1,
                    "sort": [
                      {
                        "timestamp": {
                          "order": "desc"
                        }
                      }
                    ]
                  }
                }
              }
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "def is_not_green=ctx.payload.aggregations.clusters.buckets.stream().anyMatch(t -> (t.latest_state.hits.hits[0]._source.cluster_state.status== 'yellow' || t.latest_state.hits.hits[0]._source.cluster_state.status == 'red')); if (is_not_green) { def required_periods = (ctx.metadata.not_green_secs-ctx.metadata.monitoring_update_interval)/ctx.metadata.monitoring_update_interval; return ctx.payload.aggregations.clusters.buckets.stream().anyMatch(t -> ((t.cluster_state.buckets.red.doc_count + t.cluster_state.buckets.yellow.doc_count) >= required_periods )); } return false;",
      "lang": "painless"
    }
  },
  "transform": {
    "script": {
      "source": "def required_periods = (ctx.metadata.not_green_secs-ctx.metadata.monitoring_update_interval)/ctx.metadata.monitoring_update_interval; return ctx.payload.aggregations.clusters.buckets.stream().filter(t -> (t.latest_state.hits.hits[0]._source.cluster_state.status == 'yellow' || t.latest_state.hits.hits[0]._source.cluster_state.status == 'red')).filter(t -> (t.cluster_state.buckets.red.doc_count + t.cluster_state.buckets.yellow.doc_count) >= required_periods).map(t -> ['cluster_id':t.key,'cluster_state':t.latest_state.hits.hits[0]._source.cluster_state.status,'cluster_name':t.latest_state.hits.hits[0]._source.cluster_name]).collect(Collectors.toList());",
      "lang": "painless"
    }
  },
  "actions": {
    "log": {
      "logging": {
        "text": "Clusters that have not NOT been green for more than {{ctx.metadata.not_green_secs}}s: {{#ctx.payload._value}}{{cluster_id}}-{{cluster_state}}{{/ctx.payload._value}}:"
      }
    }
  }
}