DevOpsnipp.com © 2019

Featured snippets are MIT license

Terms Of Use

Privacy Policy

Gears & Masters

  • Facebook Social Icon

SplunkAdapter.py

Splunk adapter in Python

33

Votes

# Splunk Sync SplunkAdapter
import sys
import json
import Config
from SplunkKPI import KPIQuery
import splunklib.client as client
from DataLogging import DataLogger
from  multiprocessing import Pool

logger = DataLogger.__call__().get_logger()

class SplunkAdapter(object):
    def __init__(self):
        logger.info("Creating Splunk Adapter")
        self.splunk_connection = None
        self.kpi_query_obj_list = []

    def getInfluxJSON(self):
        logger.info(str(self.__class__.__name__) + str(sys._getframe().f_code.co_name) + " : Enter")
        for kpi_query_obj in self.kpi_query_obj_list:
            logger.info("Yielding JSON Data for :" + kpi_query_obj.KPI_INFLUX_STRING)
            yield kpi_query_obj.json_value

    def generateInfluxData(self):
        logger.info(str(self.__class__.__name__) + str(sys._getframe().f_code.co_name) + " : Enter")
        for kpi_query_obj in self.kpi_query_obj_list:
            # We iterate this list of objects and then frame the response json for insertion in InfluxDB
            kpi_query_obj.generateInfluxJSON()

    def getSplunkQueryResponse(self):
        logger.info(str(self.__class__.__name__) + str(sys._getframe().f_code.co_name) + " : Enter")
        splunk_kpi_query = None
        kwargs_export  = {"output_mode" :"json"}
        # Use the python subclasses list to get the list of KPIS
        self.kpi_query_obj_list = []
        for splunk_kpi_query in KPIQuery.__subclasses__():
            if ( splunk_kpi_query.KPI_QUERY is not  None and len(splunk_kpi_query.KPI_QUERY) > 0) :
                # This is added to enable and disable queries on adhoc basis
                self.kpi_query_obj_list.append(splunk_kpi_query())

        splunk_kpi_query = None
        del splunk_kpi_query

        logger.info("Creating the splunk connection object")
        self.splunk_connection = client.connect(
            host=Config.SPLUNK_HOST,
            port=Config.SPLUNK_PORT,
            username=Config.SPLUNK_USER_NAME,
            password=Config.SPLUNK_PASSWORD
        )
        logger.info("Executing the process in parallel")
        with Pool(4) as process_pool:
            process_pool.map(execute_generate,self.kpi_query_obj_list)        

    def execute_generate(kpi_obj):
        try:
            # We execute the query which is stored as class variable for each KPI
            logger.info("Executing Query for : " + kpi_query_obj.KPI_INFLUX_STRING)
            qry_response = self.splunk_connection.jobs.oneshot( query = kpi_query_obj.KPI_QUERY , count = 0 ,**kwargs_export )
            # convert the JSON to python dict
            json_response = json.loads(qry_response.readall())
            # We check if the JSON response had a key called as reasults list
            logger.info("checking if the query response has any valid results")
            if 'results' in json_response.keys():
                # This will help us achieve two things. The part that generates the
                #influx inserts will be happy to have a empty list instead of None
                kpi_query_obj.query_result =  json_response['results']
                logger.info("Generating the JSON from the response")
                kpi_query_obj.generateInfluxJSON()
        except Exception as ex_iter:
                logger.exception("Error in iterating and querying the list of splunk kpi" + str(ex_iter))



if __name__ == "__main__":
    splunk_obj = SplunkAdapter()
    splunk_obj.getSplunkQueryResponse()
    splunk_obj.getInfluxData()

Vote Here

You must earn at least 1 vote on your snippets to be allowed to vote