cloudstack_pluginlib.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Common function for Cloudstack's XenAPI plugins

import ConfigParser
import logging
import os
import subprocess
try:
    import simplejson as json
except ImportError:
    import json
import copy

from time import localtime, asctime

DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
DEFAULT_LOG_FILE = "/var/log/cloudstack_plugins.log"

PLUGIN_CONFIG_PATH = "/etc/xensource/cloudstack_plugins.conf"
OVSDB_PID_PATH = "/var/run/openvswitch/ovsdb-server.pid"
OVSDB_DAEMON_PATH = "ovsdb-server"
OVS_PID_PATH = "/var/run/openvswitch/ovs-vswitchd.pid"
OVS_DAEMON_PATH = "ovs-vswitchd"
VSCTL_PATH = "/usr/bin/ovs-vsctl"
OFCTL_PATH = "/usr/bin/ovs-ofctl"
XE_PATH = "/opt/xensource/bin/xe"

# OpenFlow tables set in a pipeline processing fashion for the bridge created for a VPC's that are enabled for
# distributed routing.
# L2 path (intra-tier traffic)  CLASSIFIER-> L2 lookup -> L2 flooding tables
# L3 path (inter-tier traffic)  CLASSIFIER-> EGRESS ACL -> L3 lookup -> INGRESS ACL-> L2 lookup -> L2 flooding tables

# Classifier table has the rules to separate broadcast/multi-cast traffic, inter-tier traffic, intra-tier traffic
CLASSIFIER_TABLE=0
# Lookup table to determine the output port (vif/tunnel port) based on the MAC address
L2_LOOKUP_TABLE=1
# flooding table has the rules to flood on ports (both VIF, tunnel ports) except on the port on which packet arrived
L2_FLOOD_TABLE=2
# table has flow rules derived from egress ACL's
EGRESS_ACL_TABLE=3
# Lookup table to determine the output port (vif/tunnel port) based on the IP address
L3_LOOKUP_TABLE=4
# table has flow rules derived from egress ACL's
INGRESS_ACL_TABLE=5

class PluginError(Exception):
    """Base Exception class for all plugin errors."""
    def __init__(self, *args):
        Exception.__init__(self, *args)


def setup_logging(log_file=None):
    debug = False
    verbose = False
    log_format = DEFAULT_LOG_FORMAT
    log_date_format = DEFAULT_LOG_DATE_FORMAT
    # try to read plugin configuration file
    if os.path.exists(PLUGIN_CONFIG_PATH):
        config = ConfigParser.ConfigParser()
        config.read(PLUGIN_CONFIG_PATH)
        try:
            options = config.options('LOGGING')
            if 'debug' in options:
                debug = config.getboolean('LOGGING', 'debug')
            if 'verbose' in options:
                verbose = config.getboolean('LOGGING', 'verbose')
            if 'format' in options:
                log_format = config.get('LOGGING', 'format')
            if 'date_format' in options:
                log_date_format = config.get('LOGGING', 'date_format')
            if 'file' in options:
                log_file_2 = config.get('LOGGING', 'file')
        except ValueError:
            # configuration file contained invalid attributes
            # ignore them
            pass
        except ConfigParser.NoSectionError:
            # Missing 'Logging' section in configuration file
            pass

    root_logger = logging.root
    if debug:
        root_logger.setLevel(logging.DEBUG)
    elif verbose:
        root_logger.setLevel(logging.INFO)
    else:
        root_logger.setLevel(logging.WARNING)
    formatter = logging.Formatter(log_format, log_date_format)

    log_filename = log_file or log_file_2 or DEFAULT_LOG_FILE

    logfile_handler = logging.FileHandler(log_filename)
    logfile_handler.setFormatter(formatter)
    root_logger.addHandler(logfile_handler)


def do_cmd(cmd):
    """Abstracts out the basics of issuing system commands. If the command
    returns anything in stderr, a PluginError is raised with that information.
    Otherwise, the output from stdout is returned.
    """

    pipe = subprocess.PIPE
    logging.debug("Executing:%s", cmd)
    proc = subprocess.Popen(cmd, shell=False, stdin=pipe, stdout=pipe,
                            stderr=pipe, close_fds=True)
    ret_code = proc.wait()
    err = proc.stderr.read()
    if ret_code:
        logging.debug("The command exited with the error code: " +
                      "%s (stderr output:%s)" % (ret_code, err))
        raise PluginError(err)
    output = proc.stdout.read()
    if output.endswith('\n'):
        output = output[:-1]
    return output


def _is_process_run(pidFile, name):
    try:
        fpid = open(pidFile, "r")
        pid = fpid.readline()
        fpid.close()
    except IOError, e:
        return -1

    pid = pid[:-1]
    ps = os.popen("ps -ae")
    for l in ps:
        if pid in l and name in l:
            ps.close()
            return 0

    ps.close()
    return -2


def _is_tool_exist(name):
    if os.path.exists(name):
        return 0
    return -1


def check_switch():
    global result

    ret = _is_process_run(OVSDB_PID_PATH, OVSDB_DAEMON_PATH)
    if ret < 0:
        if ret == -1:
            return "NO_DB_PID_FILE"
        if ret == -2:
            return "DB_NOT_RUN"

    ret = _is_process_run(OVS_PID_PATH, OVS_DAEMON_PATH)
    if ret < 0:
        if ret == -1:
            return "NO_SWITCH_PID_FILE"
        if ret == -2:
            return "SWITCH_NOT_RUN"

    if _is_tool_exist(VSCTL_PATH) < 0:
        return "NO_VSCTL"

    if _is_tool_exist(OFCTL_PATH) < 0:
        return "NO_OFCTL"

    return "SUCCESS"


def _build_flow_expr(**kwargs):
    is_delete_expr = kwargs.get('delete', False)
    flow = ""
    if not is_delete_expr:
        flow = "hard_timeout=%s,idle_timeout=%s,priority=%s"\
                % (kwargs.get('hard_timeout', '0'),
                   kwargs.get('idle_timeout', '0'),
                   kwargs.get('priority', '1'))
    in_port = 'in_port' in kwargs and ",in_port=%s" % kwargs['in_port'] or ''
    dl_type = 'dl_type' in kwargs and ",dl_type=%s" % kwargs['dl_type'] or ''
    dl_src = 'dl_src' in kwargs and ",dl_src=%s" % kwargs['dl_src'] or ''
    dl_dst = 'dl_dst' in kwargs and ",dl_dst=%s" % kwargs['dl_dst'] or ''
    nw_src = 'nw_src' in kwargs and ",nw_src=%s" % kwargs['nw_src'] or ''
    nw_dst = 'nw_dst' in kwargs and ",nw_dst=%s" % kwargs['nw_dst'] or ''
    table = 'table' in kwargs and ",table=%s" % kwargs['table'] or ''
    cookie = 'cookie' in kwargs and ",cookie=%s" % kwargs['cookie'] or ''
    proto = 'proto' in kwargs and ",%s" % kwargs['proto'] or ''
    ip = ('nw_src' in kwargs or 'nw_dst' in kwargs) and ',ip' or ''
    flow = (flow + cookie+ in_port + dl_type + dl_src + dl_dst +
            (ip or proto) + nw_src + nw_dst + table)
    return flow


def add_flow(bridge, **kwargs):
    """
    Builds a flow expression for **kwargs and adds the flow entry
    to an Open vSwitch instance
    """
    flow = _build_flow_expr(**kwargs)
    actions = 'actions' in kwargs and ",actions=%s" % kwargs['actions'] or ''
    flow = flow + actions
    addflow = [OFCTL_PATH, "add-flow", bridge, flow]
    do_cmd(addflow)


def del_flows(bridge, **kwargs):
    """
    Removes flows according to criteria passed as keyword.
    """
    flow = _build_flow_expr(delete=True, **kwargs)
    # out_port condition does not exist for all flow commands
    out_port = ("out_port" in kwargs and
                ",out_port=%s" % kwargs['out_port'] or '')
    flow = flow + out_port
    delFlow = [OFCTL_PATH, 'del-flows', bridge, flow]
    do_cmd(delFlow)


def del_all_flows(bridge):
    delFlow = [OFCTL_PATH, "del-flows", bridge]
    do_cmd(delFlow)

    normalFlow = "priority=0 idle_timeout=0 hard_timeout=0 actions=normal"
    add_flow(bridge, normalFlow)


def del_port(bridge, port):
    delPort = [VSCTL_PATH, "del-port", bridge, port]
    do_cmd(delPort)

def get_network_id_for_vif(vif_name):
    domain_id, device_id = vif_name[3:len(vif_name)].split(".")
    hostname = do_cmd(["/bin/bash", "-c", "hostname"])
    this_host_uuid = do_cmd([XE_PATH, "host-list", "hostname=%s" % hostname, "--minimal"])
    dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "resident-on=%s" %this_host_uuid, "--minimal"])
    vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
    vnet = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid,  "param-name=other-config",
                             "param-key=cloudstack-network-id"])
    return vnet

def get_network_id_for_tunnel_port(tunnelif_name):
    vnet = do_cmd([VSCTL_PATH, "get", "interface", tunnelif_name, "options:cloudstack-network-id"])
    return vnet

def clear_flooding_rules_for_port(bridge, ofport):
        del_flows(bridge, in_port=ofport, table=L2_FLOOD_TABLE)

def clear_flooding_rules_for_all_ports(bridge):
        del_flows(bridge, cookie=111, table=L2_FLOOD_TABLE)

def add_flooding_rules_for_port(bridge, in_ofport, out_ofports):
        action = "".join("output:%s," %ofport for ofport in out_ofports)[:-1]
        add_flow(bridge, cookie=111, priority=1100, in_port=in_ofport, table=L2_FLOOD_TABLE, actions=action)

def get_ofport_for_vif(vif_name):
    return do_cmd([VSCTL_PATH, "get", "interface", vif_name, "ofport"])

def get_macaddress_of_vif(vif_name):
    domain_id, device_id = vif_name[3:len(vif_name)].split(".")
    dom_uuid = do_cmd([XE_PATH, "vm-list", "dom-id=%s" % domain_id, "--minimal"])
    vif_uuid = do_cmd([XE_PATH, "vif-list", "vm-uuid=%s" % dom_uuid, "device=%s" % device_id, "--minimal"])
    mac = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid,  "param-name=MAC"])
    return mac

def get_vif_name_from_macaddress(macaddress):
    vif_uuid = do_cmd([XE_PATH, "vif-list", "MAC=%s" % macaddress, "--minimal"])
    vif_device_id = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid,  "param-name=device"])
    vm_uuid = do_cmd([XE_PATH, "vif-param-get", "uuid=%s" % vif_uuid,  "param-name=vm-uuid"])
    vm_domain_id = do_cmd([XE_PATH, "vm-param-get", "uuid=%s" % vm_uuid,  "param-name=dom-id"])
    return "vif"+vm_domain_id+"."+vif_device_id

def add_mac_lookup_table_entry(bridge, mac_address, out_of_port):
    action = "output=%s" %out_of_port
    add_flow(bridge, priority=1100, dl_dst=mac_address, table=L2_LOOKUP_TABLE, actions=action)

def delete_mac_lookup_table_entry(bridge, mac_address):
    del_flows(bridge, dl_dst=mac_address, table=L2_LOOKUP_TABLE)

def add_ip_lookup_table_entry(bridge, ip, dst_tier_gateway_mac, dst_vm_mac):
    action_str = "mod_dl_src:%s" % dst_tier_gateway_mac + ",mod_dl_dst:%s" % dst_vm_mac + ",resubmit(,%s)"%INGRESS_ACL_TABLE
    action_str = "table=%s"%L3_LOOKUP_TABLE + ", ip, nw_dst=%s" % ip + ",  actions=%s" %action_str
    addflow = [OFCTL_PATH, "add-flow", bridge, action_str]
    do_cmd(addflow)

def get_vpc_vms_on_host(vpc, host_id):
    all_vms = vpc.vms
    vms_on_host = []
    for vm in all_vms:
      if str(vm.hostid) == (host_id):
        vms_on_host.append(vm)
    return vms_on_host

def get_network_details(vpc, network_uuid):
    tiers = vpc.tiers
    for tier in tiers:
      if str(tier.networkuuid) == (network_uuid):
        return tier
    return None

class jsonLoader(object):
  def __init__(self, obj):
        for k in obj:
            v = obj[k]
            if isinstance(v, dict):
                setattr(self, k, jsonLoader(v))
            elif isinstance(v, (list, tuple)):
                if len(v) > 0 and isinstance(v[0], dict):
                    setattr(self, k, [jsonLoader(elem) for elem in v])
                else:
                    setattr(self, k, v)
            else:
                setattr(self, k, v)

  def __getattr__(self, val):
        if val in self.__dict__:
            return self.__dict__[val]
        else:
            return None

  def __repr__(self):
        return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
                                      in self.__dict__.iteritems()))

  def __str__(self):
        return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
                                      in self.__dict__.iteritems()))
def get_acl(vpcconfig, required_acl_id):
    acls = vpcconfig.acls
    for acl in acls:
        if acl.id == required_acl_id:
            return acl
    return None

def check_tunnel_exists(bridge, tunnel_name):
    try:
        res = do_cmd([VSCTL_PATH, "port-to-br", tunnel_name])
        return res == bridge
    except:
        return False

def create_tunnel(bridge, remote_ip, gre_key, src_host, dst_host, network_uuid):

    logging.debug("Creating tunnel from host %s" %src_host + " to host %s" %dst_host + " with GRE key %s" %gre_key)

    res = check_switch()
    if res != "SUCCESS":
        logging.debug("Openvswitch running: NO")
        return "FAILURE:%s" % res

    # We need to keep the name below 14 characters
    # src and target are enough - consider a fixed length hash
    name = "t%s-%s-%s" % (gre_key, src_host, dst_host)

    # Verify the xapi bridge to be created
    # NOTE: Timeout should not be necessary anymore
    wait = [VSCTL_PATH, "--timeout=30", "wait-until", "bridge",
                    bridge, "--", "get", "bridge", bridge, "name"]
    res = do_cmd(wait)
    if bridge not in res:
        logging.debug("WARNING:Can't find bridge %s for creating " +
                                  "tunnel!" % bridge)
        return "FAILURE:NO_BRIDGE"
    logging.debug("bridge %s for creating tunnel - VERIFIED" % bridge)
    tunnel_setup = False
    drop_flow_setup = False
    try:
        # Create a port and configure the tunnel interface for it
        add_tunnel = [VSCTL_PATH, "add-port", bridge,
                                  name, "--", "set", "interface",
                                  name, "type=gre", "options:key=%s" % gre_key,
                                  "options:remote_ip=%s" % remote_ip]
        do_cmd(add_tunnel)
        tunnel_setup = True
        # verify port
        verify_port = [VSCTL_PATH, "get", "port", name, "interfaces"]
        res = do_cmd(verify_port)
        # Expecting python-style list as output
        iface_list = []
        if len(res) > 2:
            iface_list = res.strip()[1:-1].split(',')
        if len(iface_list) != 1:
            logging.debug("WARNING: Unexpected output while verifying " +
                                      "port %s on bridge %s" % (name, bridge))
            return "FAILURE:VERIFY_PORT_FAILED"

        # verify interface
        iface_uuid = iface_list[0]
        verify_interface_key = [VSCTL_PATH, "get", "interface",
                                iface_uuid, "options:key"]
        verify_interface_ip = [VSCTL_PATH, "get", "interface",
                               iface_uuid, "options:remote_ip"]

        key_validation = do_cmd(verify_interface_key)
        ip_validation = do_cmd(verify_interface_ip)

        if not gre_key in key_validation or not remote_ip in ip_validation:
            logging.debug("WARNING: Unexpected output while verifying " +
                          "interface %s on bridge %s" % (name, bridge))
            return "FAILURE:VERIFY_INTERFACE_FAILED"
        logging.debug("Tunnel interface validated:%s" % verify_interface_ip)
        cmd_tun_ofport = [VSCTL_PATH, "get", "interface",
                                          iface_uuid, "ofport"]
        tun_ofport = do_cmd(cmd_tun_ofport)
        # Ensure no trailing LF
        if tun_ofport.endswith('\n'):
            tun_ofport = tun_ofport[:-1]
        # find xs network for this bridge, verify is used for ovs tunnel network
        xs_nw_uuid = do_cmd([XE_PATH, "network-list",
								   "bridge=%s" % bridge, "--minimal"])

        ovs_tunnel_network = is_regular_tunnel_network(xs_nw_uuid)
        ovs_vpc_distributed_vr_network = is_vpc_network_with_distributed_routing(xs_nw_uuid)

        if ovs_tunnel_network == 'True':
            # add flow entryies for dropping broadcast coming in from gre tunnel
            add_flow(bridge, priority=1000, in_port=tun_ofport,
                         dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
            add_flow(bridge, priority=1000, in_port=tun_ofport,
                     nw_dst='224.0.0.0/24', actions='drop')
            drop_flow_setup = True
            logging.debug("Broadcast drop rules added")

        if ovs_vpc_distributed_vr_network == 'True':
            # add flow rules for dropping broadcast coming in from tunnel ports
            add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
                         dl_dst='ff:ff:ff:ff:ff:ff', actions='drop')
            add_flow(bridge, priority=1000, in_port=tun_ofport, table=0,
                     nw_dst='224.0.0.0/24', actions='drop')

            # add flow rule to send the traffic from tunnel ports to L2 switching table only
            add_flow(bridge, priority=1100, in_port=tun_ofport, table=0, actions='resubmit(,1)')

            # mark tunnel interface with network id for which this tunnel was created
            do_cmd([VSCTL_PATH, "set", "interface", name, "options:cloudstack-network-id=%s" % network_uuid])
            update_flooding_rules_on_port_plug_unplug(bridge, name, 'online', network_uuid)

        logging.debug("Successfully created tunnel from host %s" %src_host + " to host %s" %dst_host +
                      " with GRE key %s" %gre_key)
        return "SUCCESS:%s" % name
    except:
        logging.debug("An unexpected error occurred. Rolling back")
        if tunnel_setup:
            logging.debug("Deleting GRE interface")
            # Destroy GRE port and interface
            del_port(bridge, name)
        if drop_flow_setup:
            # Delete flows
            logging.debug("Deleting flow entries from GRE interface")
            del_flows(bridge, in_port=tun_ofport)
        # This will not cancel the original exception
        raise

# Configures the bridge created for a VPC that is enabled for distributed routing. Management server sends VPC
# physical topology details (which VM from which tier running on which host etc). Based on the VPC physical topology L2
# lookup table and L3 lookup tables are updated by this function.
def configure_vpc_bridge_for_network_topology(bridge, this_host_id, json_config, sequence_no):

    vpconfig = jsonLoader(json.loads(json_config)).vpc
    if vpconfig is None:
        logging.debug("WARNING:Can't find VPC topology information in the json configuration file")
        return "FAILURE:IMPROPER_JSON_CONFG_FILE"

    try:
        if not os.path.exists('/var/run/cloud'):
            os.makedirs('/var/run/cloud')

        # create a temporary file to store OpenFlow rules corresponding to L2 and L3 lookup table updates
        ofspec_filename = "/var/run/cloud/" + bridge + sequence_no + ".ofspec"
        ofspec = open(ofspec_filename, 'w+')

        # get the list of VM's in all the tiers of VPC running in this host from the JSON config
        this_host_vms = get_vpc_vms_on_host(vpconfig, this_host_id)

        for vm in this_host_vms:
            for nic in vm.nics:
                mac_addr = nic.macaddress
                ip = nic.ipaddress
                vif_name = get_vif_name_from_macaddress(mac_addr)
                of_port  = get_ofport_for_vif(vif_name)
                network  = get_network_details(vpconfig, nic.networkuuid)

                # Add OF rule in L2 look up table, if packet's destination mac matches MAC of the VM's nic
                # then send packet on the found OFPORT
                ofspec.write("table=%s" %L2_LOOKUP_TABLE + " priority=1100 dl_dst=%s " %mac_addr +
                             " actions=output:%s" %of_port + "\n")

                # Add OF rule in L3 look up table: if packet's destination IP matches VM's IP then modify the packet
                # to set DST MAC = VM's MAC, SRC MAC= destination tier gateway MAC and send to egress table. This step
                # emulates steps VPC virtual router would have done on the current host itself.
                action_str = " mod_dl_src:%s"%network.gatewaymac + ",mod_dl_dst:%s" % mac_addr \
                             + ",resubmit(,%s)"%INGRESS_ACL_TABLE
                action_str = "table=%s"%L3_LOOKUP_TABLE + " ip nw_dst=%s"%ip + " actions=%s" %action_str
                ofspec.write(action_str + "\n")

                # Add OF rule to send intra-tier traffic from this nic of the VM to L2 lookup path (L2 switching)
                action_str = "table=%s" %CLASSIFIER_TABLE + " priority=1200 in_port=%s " %of_port + \
                             " ip nw_dst=%s " %network.cidr + " actions=resubmit(,%s)" %L2_LOOKUP_TABLE
                ofspec.write(action_str + "\n")

                # Add OF rule to send inter-tier traffic from this nic of the VM to egress ACL table(L3 lookup path)
                action_str = "table=%s "%CLASSIFIER_TABLE + " priority=1100 in_port=%s " %of_port +\
                             " ip dl_dst=%s " %network.gatewaymac + " nw_dst=%s " %vpconfig.cidr + \
                             " actions=resubmit(,%s)" %EGRESS_ACL_TABLE
                ofspec.write(action_str + "\n")

        # get the list of hosts on which VPC spans from the JSON config
        vpc_spanning_hosts = vpconfig.hosts

        for host in vpc_spanning_hosts:
            if str(this_host_id) == str(host.hostid):
                continue

            other_host_vms = get_vpc_vms_on_host(vpconfig, str(host.hostid))

            for vm in other_host_vms:
                for nic in vm.nics:
                    mac_addr = nic.macaddress
                    ip = nic.ipaddress
                    network = get_network_details(vpconfig, nic.networkuuid)
                    gre_key = network.grekey

                    # generate tunnel name as per the tunnel naming convention
                    tunnel_name = "t%s-%s-%s" % (gre_key, this_host_id, host.hostid)

                    # check if tunnel exists already, if not create a tunnel from this host to remote host
                    if not check_tunnel_exists(bridge, tunnel_name):
                        create_tunnel(bridge, str(host.ipaddress), str(gre_key), this_host_id,
                                      host.hostid, network.networkuuid)

                    of_port = get_ofport_for_vif(tunnel_name)

                    # Add flow rule in L2 look up table, if packet's destination mac matches MAC of the VM's nic
                    # on the remote host then send packet on the found OFPORT corresponding to the tunnel
                    ofspec.write("table=%s" %L2_LOOKUP_TABLE + " priority=1100 dl_dst=%s " %mac_addr +
                                 " actions=output:%s" %of_port + "\n")

                    # Add flow rule in L3 look up table. if packet's destination IP matches VM's IP then modify the
                    # packet to set DST MAC = VM's MAC, SRC MAC=tier gateway MAC and send to ingress table. This step
                    # emulates steps VPC virtual router would have done on the current host itself.
                    action_str = "mod_dl_src:%s"%network.gatewaymac + ",mod_dl_dst:%s" % mac_addr + \
                                 ",resubmit(,%s)"%INGRESS_ACL_TABLE
                    action_str = "table=%s"%L3_LOOKUP_TABLE + " ip nw_dst=%s"%ip + " actions=%s" %action_str
                    ofspec.write(action_str + "\n")

        # add a default rule in L2_LOOKUP_TABLE to send unknown mac address to L2 flooding table
        ofspec.write("table=%s "%L2_LOOKUP_TABLE + " priority=0 " + " actions=resubmit(,%s)"%L2_FLOOD_TABLE + "\n")

        # add a default rule in L3 lookup table to forward (unknown destination IP) packets to L2 lookup table. This
        # is fallback option to send the packet to VPC VR, when routing can not be performed at the host
        ofspec.write("table=%s "%L3_LOOKUP_TABLE + " priority=0 " + " actions=resubmit(,%s)"%L2_LOOKUP_TABLE + "\n")

        # First flush current L2_LOOKUP_TABLE & L3_LOOKUP_TABLE before re-applying L2 & L3 lookup entries
        del_flows(bridge, table=L2_LOOKUP_TABLE)
        del_flows(bridge, table=L3_LOOKUP_TABLE)

        ofspec.seek(0)
        logging.debug("Adding below flows rules in L2 & L3 lookup tables:\n" + ofspec.read())
        ofspec.close()

        # update bridge with the flow-rules for L2 lookup and L3 lookup in the file in one attempt
        do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])

        # now that we updated the bridge with flow rules close and delete the file.
        os.remove(ofspec_filename)

        return "SUCCESS: successfully configured bridge as per the VPC topology update with sequence no: %s"%sequence_no

    except Exception,e:
        error_message = "An unexpected error occurred while configuring bridge " + bridge + \
                        " as per latest VPC topology update with sequence no: %s" %sequence_no
        logging.debug(error_message + " due to " + str(e))
        if os.path.isfile(ofspec_filename):
            os.remove(ofspec_filename)
        raise error_message

# Configures the bridge created for a VPC that is enabled for distributed firewall. Management server sends VPC routing
# policy (network ACL applied on the tiers etc) details. Based on the VPC routing policies ingress ACL table and
# egress ACL tables are updated by this function.
def configure_vpc_bridge_for_routing_policies(bridge, json_config, sequence_no):

    vpconfig = jsonLoader(json.loads(json_config)).vpc
    if vpconfig is None:
        logging.debug("WARNING: Can't find VPC routing policies info in json config file")
        return "FAILURE:IMPROPER_JSON_CONFG_FILE"

    try:

        if not os.path.exists('/var/run/cloud'):
            os.makedirs('/var/run/cloud')

        # create a temporary file to store OpenFlow rules corresponding to ingress and egress ACL table updates
        ofspec_filename = "/var/run/cloud/" + bridge + sequence_no + ".ofspec"
        ofspec = open(ofspec_filename, 'w+')

        tiers = vpconfig.tiers
        for tier in tiers:
            tier_cidr = tier.cidr
            acl = get_acl(vpconfig, tier.aclid)
            acl_items = acl.aclitems

            for acl_item in acl_items:
                number = acl_item.number
                action = acl_item.action
                direction = acl_item.direction
                source_port_start = acl_item.sourceportstart
                source_port_end = acl_item.sourceportend
                protocol = acl_item.protocol
                if protocol == "all":
                    protocol = "*"
                elif protocol == "tcp":
                    protocol = "6"
                elif protocol == "udp":
                    protocol == "17"
                elif protocol == "icmp":
                    protocol == "1"
                source_cidrs = acl_item.sourcecidrs
                acl_priority = 1000 + number
                if direction == "ingress":
                    matching_table = INGRESS_ACL_TABLE
                    resubmit_table = L2_LOOKUP_TABLE
                elif direction == "egress":
                    matching_table = EGRESS_ACL_TABLE
                    resubmit_table = L3_LOOKUP_TABLE

                for source_cidr in source_cidrs:
                    if source_port_start is None and source_port_end is None:
                        if source_cidr.startswith('0.0.0.0'):
                            if action == "deny":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_dst=%s " %tier_cidr + " nw_proto=%s " %protocol +
                                                 " actions=drop" + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s " %tier_cidr + " nw_proto=%s " %protocol +
                                                 " actions=drop" + "\n")
                            if action == "allow":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_dst=%s " %tier_cidr + " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s " %tier_cidr + " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table + "\n")
                        else:
                            if action == "deny":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s " %source_cidr + " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop" + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s " %tier_cidr + " nw_dst=%s " %source_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop" + "\n")
                            if action == "allow":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s "%source_cidr + " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table  + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " nw_src=%s "%tier_cidr + " nw_dst=%s " %source_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table  + "\n")
                        continue

                    # add flow rule to do action (allow/deny) for flows where source IP of the packet is in
                    # source_cidr and destination ip is in tier_cidr
                    port = int(source_port_start)
                    while (port <= int(source_port_end)):
                        if source_cidr.startswith('0.0.0.0'):
                            if action == "deny":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop"  + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop" + "\n")
                            if action == "allow":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table + "\n")
                        else:
                            if action == "deny":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s " %source_cidr +
                                                 " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop" + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s " %tier_cidr +
                                                 " nw_dst=%s " %source_cidr +
                                                 " nw_proto=%s " %protocol + " actions=drop" + "\n")
                            if action == "allow":
                                if direction == "ingress":
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s "%source_cidr +
                                                 " nw_dst=%s " %tier_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table  + "\n")
                                else:
                                    ofspec.write("table=%s "%matching_table + " priority=%s " %acl_priority + " ip " +
                                                 " tp_dst=%s " %port + " nw_src=%s "%tier_cidr +
                                                 " nw_dst=%s " %source_cidr +
                                                 " nw_proto=%s " %protocol +
                                                 " actions=resubmit(,%s)"%resubmit_table  + "\n")
                        port = port + 1

        # add a default rule in egress table to allow packets (so forward packet to L3 lookup table)
        ofspec.write("table=%s " %EGRESS_ACL_TABLE + " priority=0 actions=resubmit(,%s)" %L3_LOOKUP_TABLE + "\n")

        # add a default rule in ingress table to drop packets
        ofspec.write("table=%s " %INGRESS_ACL_TABLE + " priority=0 actions=drop" + "\n")

        # First flush current ingress and egress ACL's before re-applying the ACL's
        del_flows(bridge, table=EGRESS_ACL_TABLE)
        del_flows(bridge, table=INGRESS_ACL_TABLE)

        ofspec.seek(0)
        logging.debug("Adding below flows rules Ingress & Egress ACL tables:\n" + ofspec.read())
        ofspec.close()

        # update bridge with the flow-rules for ingress and egress ACL's added in the file in one attempt
        do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])

        # now that we updated the bridge with flow rules delete the file.
        os.remove(ofspec_filename)

        return "SUCCESS: successfully configured bridge as per the latest routing policies update with " \
               "sequence no: %s"%sequence_no

    except Exception,e:
        error_message = "An unexpected error occurred while configuring bridge " + bridge + \
                        " as per latest VPC's routing policy update with sequence number %s." %sequence_no
        logging.debug(error_message + " due to " + str(e))
        if os.path.isfile(ofspec_filename):
            os.remove(ofspec_filename)
        raise error_message

# configures bridge L2 flooding rules stored in table=2. Single bridge is used for all the tiers of VPC. So controlled
# flooding is required to restrict the broadcast to only to the ports (vifs and tunnel interfaces) in the tier. Also
# packets arrived from the tunnel ports should not be flooded on the other tunnel ports.
def update_flooding_rules_on_port_plug_unplug(bridge, interface, command, if_network_id):

    class tier_ports:
        tier_vif_ofports = []
        tier_tunnelif_ofports = []
        tier_all_ofports = []

    logging.debug("Updating the flooding rules on bridge " + bridge + " as interface  %s" %interface +
                  " is %s"%command + " now.")
    try:

        if not os.path.exists('/var/run/cloud'):
            os.makedirs('/var/run/cloud')

        # create a temporary file to store OpenFlow rules corresponding L2 flooding table
        ofspec_filename = "/var/run/cloud/" + bridge + "-" +interface + "-" + command + ".ofspec"
        ofspec = open(ofspec_filename, 'w+')

        all_tiers = dict()

        vsctl_output = do_cmd([VSCTL_PATH, 'list-ports', bridge])
        ports = vsctl_output.split('\n')

        for port in ports:

            if_ofport = do_cmd([VSCTL_PATH, 'get', 'Interface', port, 'ofport'])

            if port.startswith('vif'):
                network_id = get_network_id_for_vif(port)
                if network_id not in all_tiers.keys():
                    all_tiers[network_id] = tier_ports()
                tier_ports_info = all_tiers[network_id]
                tier_ports_info.tier_vif_ofports.append(if_ofport)
                tier_ports_info.tier_all_ofports.append(if_ofport)
                all_tiers[network_id] = tier_ports_info

            if port.startswith('t'):
                network_id = get_network_id_for_tunnel_port(port)[1:-1]
                if network_id not in all_tiers.keys():
                    all_tiers[network_id] = tier_ports()
                tier_ports_info = all_tiers[network_id]
                tier_ports_info.tier_tunnelif_ofports.append(if_ofport)
                tier_ports_info.tier_all_ofports.append(if_ofport)
                all_tiers[network_id] = tier_ports_info

        for network_id, tier_ports_info in all_tiers.items():
            if len(tier_ports_info.tier_all_ofports) == 1 :
                continue

            # for a packet arrived from tunnel port, flood only on to VIF ports connected to bridge for this tier
            for port in tier_ports_info.tier_tunnelif_ofports:
                action = "".join("output:%s," %ofport for ofport in tier_ports_info.tier_vif_ofports)[:-1]
                ofspec.write("table=%s " %L2_FLOOD_TABLE + " priority=1100 in_port=%s " %port +
                             "actions=%s " %action + "\n")

            # for a packet arrived from VIF port send on all VIF and tunnel ports corresponding to the tier excluding
            # the port on which packet arrived
            for port in tier_ports_info.tier_vif_ofports:
                tier_all_ofports_copy = copy.copy(tier_ports_info.tier_all_ofports)
                tier_all_ofports_copy.remove(port)
                action = "".join("output:%s," %ofport for ofport in tier_all_ofports_copy)[:-1]
                ofspec.write("table=%s " %L2_FLOOD_TABLE + " priority=1100 in_port=%s " %port +
                             "actions=%s " %action + "\n")

        # add a default rule in L2 flood table to drop packet
        ofspec.write("table=%s " %L2_FLOOD_TABLE + " priority=0 actions=drop")

        # First flush current L2 flooding table before re-populating the tables
        del_flows(bridge, table=L2_FLOOD_TABLE)

        ofspec.seek(0)
        logging.debug("Adding below flows rules L2 flooding table: \n" + ofspec.read())
        ofspec.close()

        # update bridge with the flow-rules for broadcast rules added in the file in one attempt
        do_cmd([OFCTL_PATH, 'add-flows', bridge, ofspec_filename])

        # now that we updated the bridge with flow rules delete the file.
        os.remove(ofspec_filename)

        logging.debug("successfully configured bridge %s as per the latest flooding rules " %bridge)

    except Exception,e:
        if os.path.isfile(ofspec_filename):
            os.remove(ofspec_filename)
        error_message = "An unexpected error occurred while updating the flooding rules for the bridge " + \
                        bridge + " when interface " + " %s" %interface + " is %s" %command
        logging.debug(error_message + " due to " + str(e))
        raise error_message


def is_regular_tunnel_network(xs_nw_uuid):
    cmd = [XE_PATH,"network-param-get", "uuid=%s" % xs_nw_uuid, "param-name=other-config",
                            "param-key=is-ovs-tun-network", "--minimal"]
    logging.debug("Executing:%s", cmd)
    pipe = subprocess.PIPE
    proc = subprocess.Popen(cmd, shell=False, stdin=pipe, stdout=pipe,
                            stderr=pipe, close_fds=True)
    ret_code = proc.wait()
    if ret_code:
        return False

    output = proc.stdout.read()
    if output.endswith('\n'):
        output = output[:-1]
    return output


def is_vpc_network_with_distributed_routing(xs_nw_uuid):
    cmd = [XE_PATH,"network-param-get", "uuid=%s" % xs_nw_uuid, "param-name=other-config",
                            "param-key=is-ovs-vpc-distributed-vr-network", "--minimal"]
    logging.debug("Executing:%s", cmd)
    pipe = subprocess.PIPE
    proc = subprocess.Popen(cmd, shell=False, stdin=pipe, stdout=pipe,
                            stderr=pipe, close_fds=True)
    ret_code = proc.wait()
    if ret_code:
        return False

    output = proc.stdout.read()
    if output.endswith('\n'):
        output = output[:-1]
    return output