init commit

.gitignore

@@ -0,0 +1 @@
+.idea/

Jenkinsfile

@@ -0,0 +1,35 @@
+pipeline {
+    agent {
+        node {
+            label 'docker'
+        }
+    }
+    stages {
+        stage('build') {
+            steps {
+                checkout  scm
+                sh 'docker-compose build'
+                sh 'docker-compose up -d'
+            }
+        }
+
+        stage('tests') {
+            steps {
+                sh 'python app.py'
+            }
+        }
+    }
+    post {
+        always {
+            sh "docker-compose down"
+            deleteDir()
+        }
+        success {
+            echo 'Test Successful'
+        }
+
+        failure {
+            echo 'Test failed'
+        }
+    }
+}

README.md

@@ -0,0 +1,12 @@
+* Create file <i><template_name>.json</i> and fill it with ES index template
+* Copy <i><template_name>.json</i>  into <i>templates</i> directory
+* Create file <i><template_name>.json</i> and fill it with sample data for your <i><template_name></i>  
+* Copy <i><template_name>.json</i>  into <i>samples</i> directory
+
+* Build and run ES container:<br>
+<code>docker-compose up -d</code>
+
+* Run app.py script and read it's output:<br>
+<code>python app.py</code>
+
+If ES-generated mapping matches your template script will exit with 0 status code or 1 in opposite case.

app.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python2.7
+# -*- coding: utf-8 -*-
+
+import requests
+import os
+import sys
+import json
+import time
+
+TMPL_DIR = 'templates'
+SMPL_DIR = 'samples'
+ES_HOST = 'localhost'
+ES_PORT = '9200'
+
+# Obtaining list of templates
+templates = os.listdir(TMPL_DIR)
+
+
+# Waiting while ES starts
+def check_es_ready():
+    tries = 0
+    while tries < 10:
+        try:
+            requests.get('http://%s:%s' % (ES_HOST, ES_PORT), timeout=3)
+            return
+        except requests.exceptions.ConnectionError:
+            tries+=1
+            print "ES not ready yet. Attempting to reconnect (#%s)" % tries
+            time.sleep(3)
+    print "ES not ready. Retry limit reached. Aborting script"
+    sys.exit(1)
+
+
+# Main test
+def perform_check(item):
+    index_name = item.split('.')[0]
+    # No need to run if no sample data for template
+    if item not in os.listdir(SMPL_DIR):
+        print "No sample data for index %s. Aborting" % index_name
+        sys.exit(1)
+
+    # Obtaining existing indices from test ES
+    try:
+        req = requests.get('http://%s:%s/_cat/indices?h=index' % (ES_HOST, ES_PORT))
+        if req.status_code == 200:
+            indices = req.text.split('\n')
+        else:
+            print 'Failed to obtain indeces: %s' % req.text
+            sys.exit(1)
+    except requests.exceptions.ConnectionError as e:
+        print 'Failed to connect to ES', e
+        sys.exit(1)
+
+    # Deleting existing index if exists
+    if index_name in indices:
+        try:
+            req = requests.delete('http://%s:%s/%s' % (ES_HOST, ES_PORT, index_name))
+            if req.status_code == 200:
+                response = json.loads(req.text)
+                print 'Deleting index "%s": %s' % (index_name, response["acknowledged"])
+            else:
+                print 'Failed to clear index "%s": %s' % (index_name, req.text)
+                sys.exit(1)
+        except requests.exceptions.ConnectionError as e:
+            print 'Failed to connect to ES', e
+            sys.exit(1)
+
+    # Creating new template for index
+    with open('%s/%s' % (TMPL_DIR, item), 'r') as template:
+        try:
+            mapping = json.loads((template.read()))
+        except ValueError:
+            print 'Failed to serialize template for index "%s"' % index_name
+            sys.exit(1)
+
+        try:
+            req = requests.put('http://%s:%s/_template/%s' % (ES_HOST, ES_PORT, index_name), json=mapping)
+            if req.status_code == 200:
+                response = json.loads(req.text)
+                print 'Creating template for index "%s": %s' % (index_name, response["acknowledged"])
+            else:
+                print 'Failed to create template for index "%s": %s' % (index_name, req.text)
+        except requests.exceptions.ConnectionError as e:
+            print 'Failed to connect to ES', e
+            sys.exit(1)
+
+    # Posting sample data into index
+    with open('%s/%s' % (SMPL_DIR, item), 'r') as sample:
+        try:
+            data = json.loads(sample.read())
+        except ValueError:
+            print 'Failed to serialize sample data for index "%s"' % index_name
+            sys.exit(1)
+        try:
+            req = requests.post('http://%s:%s/%s/_doc' % (ES_HOST, ES_PORT, index_name), json=data)
+            if req.status_code == 201:
+                response = json.loads(req.text)
+                print 'Posting sample data to index "%s": %s' % (index_name, response["result"])
+            else:
+                print 'Failed to post sample data to index "%s": %s' % (index_name, req.text)
+        except requests.exceptions.ConnectionError as e:
+            print 'Failed to connect to ES', e
+            sys.exit(1)
+
+    # Obtaining autocreated mapping for index and comparing it with template
+    try:
+        req = requests.get('http://%s:%s/%s/_mapping' % (ES_HOST, ES_PORT, index_name), json=mapping)
+    except requests.exceptions.ConnectionError as e:
+        print 'Failed to connect to ES', e
+        sys.exit(1)
+    cur_mapping = json.loads(req.text)
+    compare_result = cmp(mapping["mappings"], cur_mapping[index_name]["mappings"])
+    if compare_result == 1:
+        print 'Template for index "%s" sucessfully passed the test\n\n' % index_name
+        return
+    else:
+        print 'Generated template is differ from original template for index "%s"' % index_name
+
+
+if __name__ == "__main__":
+    # Waiting for elasticsearch to start
+    check_es_ready()
+    # For each template execute series of tests:
+    for entry in templates:
+        perform_check(entry)
+    print 'All tests passed'
+    sys.exit(0)

docker-compose.yml

@@ -0,0 +1,15 @@
+version: '2.2'
+services:
+  es-test:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.1
+    container_name: es-test
+    environment:
+      - cluster.name=es-test-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    ports:
+      - 9200:9200

samples/2test.json

@@ -0,0 +1,9 @@
+{
+    "long" :  9223372036854775807,
+    "message" : "Thank you Sabuj, I have instal CURL and run the code, it create in 'data' folder a indexname. But anytime I need to insert data to ElasticSearch, I must use these tool like CURL, RESTClient... As my understand, ElasticSearch use webservice to contact with data, am I right? In the case I need to interact with Elastic using my application, how can I do it? Many thanks – user1162069",
+    "integer" : 214748364,
+    "@timestamp" : "2019-12-02T13:33:54.974Z",
+    "object" : {"testobj": {"bla": 1}},
+    "short" : 5,
+    "float" : 12.15
+}

samples/test.json

@@ -0,0 +1,9 @@
+{
+    "long" :  9223372036854775807,
+    "message" : "Thank you Sabuj, I have instal CURL and run the code, it create in 'data' folder a indexname. But anytime I need to insert data to ElasticSearch, I must use these tool like CURL, RESTClient... As my understand, ElasticSearch use webservice to contact with data, am I right? In the case I need to interact with Elastic using my application, how can I do it? Many thanks – user1162069",
+    "integer" : 214748364,
+    "@timestamp" : "2019-12-02T13:33:54.974Z",
+    "object" : {"testobj": {"bla": 1}},
+    "short" : 5,
+    "float" : 12.15
+}

templates/2test.json

@@ -0,0 +1,48 @@
+{
+    "version" : 10,
+    "index_patterns" : [
+      "test*"
+    ],
+    "settings" : {
+      "index" : {
+        "refresh_interval" : "30s",
+        "number_of_shards" : "6",
+        "number_of_replicas" : "2"
+      }
+    },
+    "mappings" : {
+      "_doc" : {
+        "properties" : {
+          "long" : {
+            "type" : "long"
+          },
+          "message" : {
+            "norms" : false,
+            "index" : true,
+            "type" : "text"
+          },
+          "integer" : {
+            "index" : true,
+            "type" : "integer"
+          },
+          "@timestamp" : {
+            "type" : "date",
+            "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'||epoch_millis"
+          },
+          "object" : {
+            "type" : "object",
+            "enabled" : false
+          },
+          "short" : {
+            "index" : true,
+            "type" : "short"
+          },
+          "float" : {
+            "index": true,
+            "type" : "float"
+          }
+        }
+      }
+    },
+    "aliases" : { }
+}

templates/test.json

@@ -0,0 +1,48 @@
+{
+    "version" : 10,
+    "index_patterns" : [
+      "test*"
+    ],
+    "settings" : {
+      "index" : {
+        "refresh_interval" : "30s",
+        "number_of_shards" : "6",
+        "number_of_replicas" : "2"
+      }
+    },
+    "mappings" : {
+      "_doc" : {
+        "properties" : {
+          "long" : {
+            "type" : "long"
+          },
+          "message" : {
+            "norms" : false,
+            "index" : true,
+            "type" : "text"
+          },
+          "integer" : {
+            "index" : true,
+            "type" : "integer"
+          },
+          "@timestamp" : {
+            "type" : "date",
+            "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'||epoch_millis"
+          },
+          "object" : {
+            "type" : "object",
+            "enabled" : false
+          },
+          "short" : {
+            "index" : true,
+            "type" : "short"
+          },
+          "float" : {
+            "index": true,
+            "type" : "float"
+          }
+        }
+      }
+    },
+    "aliases" : { }
+}