Machine Learning

Créer des jobs de détection

Configuration du job sur les données de stockage

PUT _ml/anomaly_detectors/ml-nodes-fs-free
{
  "job_id": "ml-nodes-fs-free",
  "groups": [
    "ml-default"
  ],
  "description": "",
  "analysis_config": {
    "bucket_span": "15m",
    "detectors": [
      {
        "detector_description": "mean(\"cluster_stats.nodes.fs.free_in_bytes\") by \"cluster_settings.cluster.metadata.display_name\"",
        "function": "mean",
        "field_name": "cluster_stats.nodes.fs.free_in_bytes",
        "by_field_name": "cluster_settings.cluster.metadata.display_name",
        "custom_rules": [
          {
            "actions": [
              "skip_result"
            ],
            "conditions": [
              {
                "applies_to": "diff_from_typical",
                "operator": "lte",
                "value": 3000000000
              }
            ]
          }
        ],
        "detector_index": 0
      }
    ],
    "influencers": [
      "cluster_settings.cluster.metadata.display_name"
    ]
  },
  "analysis_limits": {
    "model_memory_limit": "315mb",
    "categorization_examples_limit": 4
  },
  "data_description": {
    "time_field": "timestamp",
    "time_format": "epoch_ms"
  },
  "model_plot_config": {
    "enabled": false,
    "annotations_enabled": true
  },
  "model_snapshot_retention_days": 10,
  "daily_model_snapshot_retention_after_days": 1,
  "results_index_name": "shared",
  "allow_lazy_open": false,
  "datafeed_config": {
    "datafeed_id": "datafeed-ml-nodes-fs-free",
    "job_id": "ml-nodes-fs-free",
    "query_delay": "60s",
    "chunking_config": {
      "mode": "auto"
    },
    "indices_options": {
      "expand_wildcards": [
        "open"
      ],
      "ignore_unavailable": false,
      "allow_no_indices": true,
      "ignore_throttled": true
    },
    "query": {
      "bool": {
        "must": [
          {
            "match": {
              "type": "cluster_stats"
            }
          }
        ]
      }
    },
    "frequency": "450s",
    "indices": [
      ".monitoring-es-7-*"
    ],
    "scroll_size": 10000,
    "delayed_data_check_config": {
      "enabled": true
    }
  }
}

Ouvrir le job

POST _ml/anomaly_detectors/ml-nodes-fs-free/_open

Démarrer du datafeed

POST _ml/datafeeds/datafeed-ml-nodes-fs-free/_start

Synchronisation du job

curl --location --request GET 'https://cluster.example.com/api/ml/saved_objects/sync' \
--header 'kbn-xsrf: true' \
--header 'Authorization: Basic ***********'

Configuration du job sur les données de JVM

PUT _ml/anomaly_detectors/ml-jvm-mem-heap-used
{
  "job_id": "ml-jvm-mem-heap-used",
  "groups": [
    "ml-default"
  ],
  "description": "",
  "analysis_config": {
    "bucket_span": "15m",
    "detectors": [
      {
        "detector_description": "mean(\"cluster_stats.nodes.jvm.mem.heap_used_in_bytes\") by \"cluster_settings.cluster.metadata.display_name\"",
        "function": "mean",
        "field_name": "cluster_stats.nodes.jvm.mem.heap_used_in_bytes",
        "by_field_name": "cluster_settings.cluster.metadata.display_name",
        "detector_index": 0
      }
    ],
    "influencers": [
      "cluster_settings.cluster.metadata.display_name"
    ]
  },
  "analysis_limits": {
    "model_memory_limit": "312mb",
    "categorization_examples_limit": 4
  },
  "data_description": {
    "time_field": "timestamp",
    "time_format": "epoch_ms"
  },
  "model_plot_config": {
    "enabled": false,
    "annotations_enabled": false
  },
  "model_snapshot_retention_days": 10,
  "daily_model_snapshot_retention_after_days": 1,
  "results_index_name": "shared",
  "allow_lazy_open": false,
  "datafeed_config": {
    "datafeed_id": "datafeed-ml-jvm-mem-heap-used",
    "job_id": "ml-jvm-mem-heap-used",
    "query_delay": "60s",
    "query": {
      "bool": {
        "must": [
          {
            "match": {
              "type": "cluster_stats"
            }
          }
        ]
      }
    },
    "frequency": "450s",
    "indices": [
      ".monitoring-es-7-*"
    ],
    "scroll_size": 10000,
    "delayed_data_check_config": {
      "enabled": true
    }
  }
}

Ouvrir le job

POST _ml/anomaly_detectors/ml-jvm-mem-heap-used/_open

Démarrer du datafeed

POST _ml/datafeeds/datafeed-ml-jvm-mem-heap-used/_start

Synchronisation du job

curl --location --request GET 'https://cluster.example.com/api/ml/saved_objects/sync' \
--header 'kbn-xsrf: true' \
--header 'Authorization: Basic ***********'

Configuration du job sur les données de disponibilité (uptime)

PUT _ml/anomaly_detectors/ml-uptime-url-elisa
{
  "job_id": "ml-uptime-url-elisa",
  "datafeed_config": {
    "datafeed_id": "datafeed-ml-uptime-url-elisa",
    "job_id": "ml-uptime-url-elisa",
    "query_delay": "93743ms",
    "query": {
      "bool": {
        "must": [
          {
            "match_all": {}
          }
        ]
      }
    },
    "indices": [
      "heartbeat-*"
    ],
    "scroll_size": 10000,
    "delayed_data_check_config": {
      "enabled": true
    }
  },
  "groups": [
    "ml-default",
    "uptime"
  ],
  "description": "",
  "analysis_config": {
    "bucket_span": "15m",
    "detectors": [
      {
        "detector_description": "high_mean(\"monitor.duration.us\") by \"url.full\"",
        "function": "high_mean",
        "field_name": "monitor.duration.us",
        "by_field_name": "url.full",
        "detector_index": 0
      }
    ],
    "influencers": [
      "url.full"
    ]
  },
  "analysis_limits": {
    "model_memory_limit": "315mb",
    "categorization_examples_limit": 4
  },
  "data_description": {
    "time_field": "@timestamp",
    "time_format": "epoch_ms"
  },
  "model_plot_config": {
    "enabled": false,
    "annotations_enabled": false
  },
  "model_snapshot_retention_days": 10,
  "daily_model_snapshot_retention_after_days": 1,
  "results_index_name": "shared",
  "allow_lazy_open": false
}

Ouvrir le job

POST _ml/anomaly_detectors/ml-uptime-url-elisa/_open

Démarrer du datafeed

POST _ml/datafeeds/datafeed-ml-uptime-url-elisa/_start

Synchronisation du job

curl --location --request GET 'https://cluster.example.com/api/ml/saved_objects/sync' \
--header 'kbn-xsrf: true' \
--header 'Authorization: Basic ***********'

Mettre à jour un job detection

Exemple sur la limitation de mémoire pour le job de stockage

Vérifié la présence de job détection

GET _ml/anomaly_detectors/

Stopper le job détection

POST _ml/anomaly_detectors/ml-nodes-fs-free/_close

Mise à jour le job détection

POST _ml/anomaly_detectors/ml-nodes-fs-free/_update
{
  "analysis_limits" : {
        "model_memory_limit" : "500mb",
        "categorization_examples_limit" : 4
      }
}

Ouvrir le job détection

POST _ml/anomaly_detectors/ml-nodes-fs-free/_open

Démarrage du datafeed

POST _ml/datafeeds/datafeed-ml-nodes-fs-free/_start

Vérifié la mise à jour du job détection

GET _ml/anomaly_detectors/

Exemple sur la limitation de mémoire pour le job de JVM

GET _ml/anomaly_detectors/

Stopper le job détection

POST _ml/anomaly_detectors/ml-jvm-mem-heap-used/_close

Mise à jour le job détection

POST _ml/anomaly_detectors/ml-jvm-mem-heap-used/_update
{
  "analysis_limits" : {
        "model_memory_limit" : "500mb",
        "categorization_examples_limit" : 4
      }
}

Ouvrir le job détection

POST _ml/anomaly_detectors/ml-jvm-mem-heap-used/_open

Démarrage du datafeed

POST _ml/datafeeds/datafeed-ml-jvm-mem-heap-used/_start

Vérifié la mise à jour du job détection

GET _ml/anomaly_detectors/