- Created by Jean-luc KRIKER, last modified on Jul 14, 2020
You are viewing an old version of this page. View the current version.
Compare with Current View Page History
« Previous Version 5 Next »
blueprint: NorthStar (v5) and HealthBot (v2.0.2) | |
video | |
NortStar or NS login | https://<ip@><port> admin // Juniper!1 |
HealthBot | SSH or https:// jcluser // Juniper!1 |
healthbot_listener.py |
HealthBot Listener script Expand source
# We need to import request to access the details of the POST request from flask import Flask, request from flask_restful import abort import commands import json import pprint import requests import os import user_functions requests.packages.urllib3.disable_warnings() # Initialize the Flask application app = Flask(__name__) @app.route('/', methods=['POST']) def app_message_post(): print "################# Start #######################" if request.headers['Content-Type'] != 'application/json': abort(400, message="Expected Content-Type = application/json") try: data = request.json print data message = data['message'] print "message " + message playbook_name = data['keys']['_playbook_name'] print "playbook_name " + playbook_name #test_name = data['keys']['test-name'] #print "interface-name " + test_name device_id = data['device-id'] #print device_id + " " + message #if spec['eventRuleId'] == g_rule_id: # state = status['state'] # device_id = status['entityId'] # if state == "active" and device_id == g_device_id: # print 'DATA_ACTIVE :: ', pprint.pprint(data) # user_functions.move_traffic() # print 'traffic detoured and Slack was notified' # elif state == "inactive": # #print 'DATA_INACTIVE :: ', pprint.pprint(data) # print 'LSP path can be changed back' #return json.dumps({'result': 'OK'}) if playbook_name == "cpu_openconfig": print "received cpu high alert" if "exceeds high threshold" in message: print 'CPU HIGH UTIL DETECTED for ' + device_id print 'PERFORMING EXHUASTIVE LINK FAILURE SIMULATION for ' + device_id #create maintenance for simulation purpose rest_index_number = user_functions.get_node_info(device_id) rest_payload = user_functions.generate_maitenance_json(rest_index_number, 'for_simulation', 'node') maintenance_event = user_functions.create_maintenance(rest_payload) maintenance_index = maintenance_event.json()['maintenanceIndex'] check_simulation = user_functions.check_if_simulation_pass() print "simulation result " + check_simulation user_functions.delete_maintenance(maintenance_index) print "delete temp maintenace" if check_simulation == 'true': print 'CPU HIGH UTIL DETECTED PUT NODE UNDER MAINTENANCE::' # pprint.pprint(data) #print "rest_node_name, rest_index_number" + rest_node_name + rest_index_number rest_payload = user_functions.generate_maitenance_json(rest_index_number, 'for_maint', 'node') print rest_payload user_functions.create_maintenance(rest_payload) else: print 'CANNOT PUT ' + device_id + ' UNDER MAINTENANCE. EXHUASTIVE FAILURE SIMULATION NOT PASSED' elif "is normal" in message: #print 'DATA_INACTIVE :: ', pprint.pprint(data) print 'CPU util back to normal. ' print '###############################' if playbook_name == "delay": print "received delay alert" source_address = data['keys']['source-address'] #print "interface-ip " + source_address #target_address = data['keys']['target_address'] #print "message" + message if "exceeds delay threshold" in message: print "HIGH DELAY DETECTED for " + device_id + " " + source_address print "PERFORMING EXHUASTIVE LINK FAILURE SIMULATION for " + device_id + " " + source_address #create maintenance for simulation purpose rest_index_number = user_functions.get_link_info_from_ip(source_address) rest_payload = user_functions.generate_maitenance_json(rest_index_number, 'for_simulation', 'link') maintenance_event = user_functions.create_maintenance(rest_payload) maintenance_index = maintenance_event.json()['maintenanceIndex'] check_simulation = user_functions.check_if_simulation_pass() print "SIMULATION RESULT " + check_simulation user_functions.delete_maintenance(maintenance_index) #print "delete temp maintenace" if check_simulation == "true": print "HIGH DELAY DETECTED PUT LINK UNDER MAINTENANCE::" # pprint.pprint(data) #print "rest_node_name, rest_index_number" + rest_node_name + rest_index_number rest_payload = user_functions.generate_maitenance_json(rest_index_number, 'for_maint', 'link') print rest_payload user_functions.create_maintenance(rest_payload) else: print "CANNOT PUT " + device_id + " " + source_address + " UNDER MAINTENANCE. EXHUASTIVE FAILURE SIMULATION NOT PASSED" elif "is normal" in message: #print 'DATA_INACTIVE :: ', pprint.pprint(data) print "DELAY back to normal. " print "###############################" """ if event_rule_id == AppFormixInterfaceL3IncompleteEventID: print "Received interface l3 incomplete alert" if state == "active": rest_payload = user_functions.generate_link_maitenance_json() print rest_payload user_functions.create_maintenance(rest_payload) print 'Put problematic link into maintenance mode' elif state == "inactive": # print 'DATA_INACTIVE :: ', pprint.pprint(data) print 'link back to normal. you can complete the maintenance event' """ return json.dumps({'result': 'OK'}) except Exception as e: abort(400, message="Exception processing request: {0}".format(e)) print '...' if __name__ == '__main__': app.run( host="0.0.0.0", port=int("10000") ) |
add delay | Log into the CentOS ( jcluser / Juniper!1 ) [root@CentOS ~]# more add_delay.sh |
healthbot_listener.py output |
output Expand source
jcluser@ubuntu:~/self_healing$ python healthbot_listener.py /usr/local/lib/python2.7/dist-packages/urllib3/connectionpool.py:1004: InsecureRequestWarning: Unverified HTTPS request is being ma de. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warn ings InsecureRequestWarning, * Serving Flask app "healthbot_listener" (lazy loading) * Environment: production WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Debug mode: off * Running on http://0.0.0.0:10000/ (Press CTRL+C to quit) ################# Start ####################### {u'group': u'group_all', u'severity': u'major', u'keys': {u'_instance_id': u'["delay"]', u'source-address': u'7.105.106.1', u'_play book_name': u'delay'}, u'device-id': u'vMX-5', u'rule': u'probe_delay', u'topic': u'probe-delay', u'trigger': u'probe_exceed', u'me ssage': u'7.105.106.1 ge-0/0/3.0 delay is 303099 exceeds delay threshold 200000 us. '} message 7.105.106.1 ge-0/0/3.0 delay is 303099 exceeds delay threshold 200000 us. playbook_name delay ############################### received delay alert HIGH DELAY DETECTED for vMX-5 7.105.106.1 PERFORMING EXHUASTIVE LINK FAILURE SIMULATION for vMX-5 7.105.106.1 { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "created_for_simulation", "startTime": "20200716T230300Z", "endTime": "20200718T150300Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } SIMULATION RESULT true HIGH DELAY DETECTED PUT LINK UNDER MAINTENANCE:: { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "Healthbot-link-health-alert202007141103", "startTime": "20200714T110400Z", "endTime": "20200718T150300Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "Healthbot-link-health-alert202007141103", "startTime": "20200714T110400Z", "endTime": "20200718T150300Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } ############################### 172.22.0.3 - - [14/Jul/2020 04:03:29] "POST / HTTP/1.1" 200 - ################# Start ####################### {u'group': u'group_all', u'severity': u'major', u'keys': {u'_instance_id': u'["delay"]', u'source-address' : u'7.105.106.2', u'_playbook_name': u'delay'}, u'device-id': u'vMX-6', u'rule': u'probe_delay', u'topic': u'probe-delay', u'trigger': u'probe_exceed', u'message': u'7.105.106.2 ge-0/0/3.0 delay is 274028 exceed s delay threshold 200000 us. '} message 7.105.106.2 ge-0/0/3.0 delay is 274028 exceeds delay threshold 200000 us. playbook_name delay ############################### received delay alert HIGH DELAY DETECTED for vMX-6 7.105.106.2 PERFORMING EXHUASTIVE LINK FAILURE SIMULATION for vMX-6 7.105.106.2 { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "created_for_simulation", "startTime": "20200716T230400Z", "endTime": "20200718T150400Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } SIMULATION RESULT true HIGH DELAY DETECTED PUT LINK UNDER MAINTENANCE:: { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "Healthbot-link-health-alert202007141104", "startTime": "20200714T110500Z", "endTime": "20200718T150400Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } { "topoObjectType": "maintenance", "topologyIndex": 1, "user": "admin", "name": "Healthbot-link-health-alert202007141104", "startTime": "20200714T110500Z", "endTime": "20200718T150400Z", "elements": [ { "topoObjectType": "link", "index": 9 } ] } ############################### 172.22.0.3 - - [14/Jul/2020 04:04:29] "POST / HTTP/1.1" 200 - |
- No labels