1 # Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
3 # This is based on the following
4 # https://github.com/osrg/gobgp/test/lib/base.py
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 from __future__ import absolute_import
30 LOG = logging.getLogger(__name__)
32 DEFAULT_TEST_PREFIX = ''
33 DEFAULT_TEST_BASE_DIR = '/tmp/ctn_docker/bgp'
34 TEST_PREFIX = DEFAULT_TEST_PREFIX
35 TEST_BASE_DIR = DEFAULT_TEST_BASE_DIR
37 BGP_FSM_IDLE = 'BGP_FSM_IDLE'
38 BGP_FSM_ACTIVE = 'BGP_FSM_ACTIVE'
39 BGP_FSM_ESTABLISHED = 'BGP_FSM_ESTABLISHED'
41 BGP_ATTR_TYPE_ORIGIN = 1
42 BGP_ATTR_TYPE_AS_PATH = 2
43 BGP_ATTR_TYPE_NEXT_HOP = 3
44 BGP_ATTR_TYPE_MULTI_EXIT_DISC = 4
45 BGP_ATTR_TYPE_LOCAL_PREF = 5
46 BGP_ATTR_TYPE_COMMUNITIES = 8
47 BGP_ATTR_TYPE_ORIGINATOR_ID = 9
48 BGP_ATTR_TYPE_CLUSTER_LIST = 10
49 BGP_ATTR_TYPE_MP_REACH_NLRI = 14
50 BGP_ATTR_TYPE_EXTENDED_COMMUNITIES = 16
52 BRIDGE_TYPE_DOCKER = 'docker'
53 BRIDGE_TYPE_BRCTL = 'brctl'
54 BRIDGE_TYPE_OVS = 'ovs'
57 class CommandError(Exception):
58 def __init__(self, out):
59 super(CommandError, self).__init__()
63 def try_several_times(f, t=3, s=1):
68 except RuntimeError as e:
75 class CmdBuffer(list):
76 def __init__(self, delim='\n'):
77 super(CmdBuffer, self).__init__()
80 def __lshift__(self, value):
84 return self.delim.join(self)
87 class CommandOut(str):
89 def __new__(cls, stdout, stderr, command, returncode, **kwargs):
91 obj = super(CommandOut, cls).__new__(cls, stdout, **kwargs)
92 obj.stderr = stderr or ''
94 obj.returncode = returncode
98 class Command(object):
100 def _execute(self, cmd, capture=False, executable=None):
101 """Execute a command using subprocess.Popen()
103 - out: stdout from subprocess.Popen()
104 out has some attributes.
105 out.returncode: returncode of subprocess.Popen()
106 out.stderr: stderr from subprocess.Popen()
109 p_stdout = subprocess.PIPE
110 p_stderr = subprocess.PIPE
114 pop = subprocess.Popen(cmd, shell=True, executable=executable,
117 __stdout, __stderr = pop.communicate()
118 _stdout = six.text_type(__stdout, 'utf-8')
119 _stderr = six.text_type(__stderr, 'utf-8')
120 out = CommandOut(_stdout, _stderr, cmd, pop.returncode)
123 def execute(self, cmd, capture=True, try_times=1, interval=1):
125 for i in range(try_times):
126 out = self._execute(cmd, capture=capture)
127 LOG.info(out.command)
128 if out.returncode == 0:
130 LOG.error("stdout: %s", out)
131 LOG.error("stderr: %s", out.stderr)
132 if i + 1 >= try_times:
135 raise CommandError(out)
137 def sudo(self, cmd, capture=True, try_times=1, interval=1):
138 cmd = 'sudo %s' % cmd
139 return self.execute(cmd, capture=capture,
140 try_times=try_times, interval=interval)
143 class DockerImage(object):
144 def __init__(self, baseimage='ubuntu:16.04'):
145 self.baseimage = baseimage
148 def get_images(self):
149 out = self.cmd.sudo('sudo docker images')
151 for line in out.splitlines()[1:]:
152 images.append(line.split()[0])
155 def exist(self, name):
156 return name in self.get_images()
158 def build(self, tagname, dockerfile_dir):
160 "docker build -t {0} {1}".format(tagname, dockerfile_dir),
163 def remove(self, tagname, check_exist=False):
164 if check_exist and not self.exist(tagname):
166 self.cmd.sudo("docker rmi -f %s" % tagname, try_times=3)
168 def create_quagga(self, tagname='quagga', image=None, check_exist=False):
169 if check_exist and self.exist(tagname):
171 workdir = os.path.join(TEST_BASE_DIR, tagname)
180 use_image = self.baseimage
182 c << 'FROM %s' % use_image
183 c << 'RUN apt-get update'
184 c << 'RUN apt-get install -qy --no-install-recommends %s' % pkges
185 c << 'CMD /usr/lib/quagga/bgpd'
187 self.cmd.sudo('rm -rf %s' % workdir)
188 self.cmd.execute('mkdir -p %s' % workdir)
189 self.cmd.execute("echo '%s' > %s/Dockerfile" % (str(c), workdir))
190 self.build(tagname, workdir)
193 def create_ryu(self, tagname='ryu', image=None, check_exist=False):
194 if check_exist and self.exist(tagname):
196 workdir = os.path.join(TEST_BASE_DIR, tagname)
197 workdir_ctn = '/root/osrg/ryu'
205 use_image = self.baseimage
207 c << 'FROM %s' % use_image
208 c << 'ADD ryu %s' % workdir_ctn
210 'RUN apt-get update',
211 '&& apt-get install -qy --no-install-recommends %s' % pkges,
212 '&& cd %s' % workdir_ctn,
213 # Note: Clean previous builds, because "python setup.py install"
214 # might fail if the current directory contains the symlink to
215 # Docker host file systems.
216 '&& rm -rf *.egg-info/ build/ dist/ .tox/ *.log'
217 '&& pip install -r tools/pip-requires -r tools/optional-requires',
218 '&& python setup.py install',
222 self.cmd.sudo('rm -rf %s' % workdir)
223 self.cmd.execute('mkdir -p %s' % workdir)
224 self.cmd.execute("echo '%s' > %s/Dockerfile" % (str(c), workdir))
225 self.cmd.execute('cp -r ../ryu %s/' % workdir)
226 self.build(tagname, workdir)
230 class Bridge(object):
231 def __init__(self, name, subnet='', start_ip=None, end_ip=None,
232 with_ip=True, self_ip=False,
233 fixed_ip=None, reuse=False,
238 - subnet: network cider to be used in this bridge
239 - start_ip: start address of an ip to be used in the subnet
240 - end_ip: end address of an ip to be used in the subnet
241 - with_ip: specify if assign automatically an ip address
242 - self_ip: specify if assign an ip address for the bridge
243 - fixed_ip: an ip address to be assigned to the bridge
244 - reuse: specify if use an existing bridge
245 - br_type: One either in a 'docker', 'brctl' or 'ovs'
249 if br_type not in (BRIDGE_TYPE_DOCKER, BRIDGE_TYPE_BRCTL,
251 raise Exception("argument error br_type: %s" % br_type)
252 self.br_type = br_type
253 self.docker_nw = bool(self.br_type == BRIDGE_TYPE_DOCKER)
254 if TEST_PREFIX != '':
255 self.name = '{0}_{1}'.format(TEST_PREFIX, name)
256 self.with_ip = with_ip
258 self.subnet = netaddr.IPNetwork(subnet)
260 self.start_ip = start_ip
262 self.start_ip = netaddr.IPAddress(self.subnet.first)
266 self.end_ip = netaddr.IPAddress(self.subnet.last)
269 for host in netaddr.IPRange(self.start_ip, self.end_ip):
271 self._ip_generator = _ip_gen()
272 # throw away first network address
273 self.next_ip_address()
275 self.self_ip = self_ip
277 self.ip_addr = fixed_ip
279 self.ip_addr = self.next_ip_address()
282 if self.br_type == BRIDGE_TYPE_DOCKER:
283 gw = "--gateway %s" % self.ip_addr.split('/')[0]
285 if self.subnet.version == 6:
287 cmd = ("docker network create --driver bridge %s "
288 "%s --subnet %s %s" % (v6, gw, subnet, self.name))
289 elif self.br_type == BRIDGE_TYPE_BRCTL:
290 cmd = "ip link add {0} type bridge".format(self.name)
291 elif self.br_type == BRIDGE_TYPE_OVS:
292 cmd = "ovs-vsctl add-br {0}".format(self.name)
294 raise ValueError('Unsupported br_type: %s' % self.br_type)
296 self.execute(cmd, sudo=True, retry=True)
298 if not self.docker_nw:
299 self.execute("ip link set up dev {0}".format(self.name),
300 sudo=True, retry=True)
302 if not self.docker_nw and self_ip:
303 ips = self.check_br_addr(self.name)
304 for key, ip in ips.items():
305 if self.subnet.version == key:
307 "ip addr del {0} dev {1}".format(ip, self.name),
308 sudo=True, retry=True)
310 "ip addr add {0} dev {1}".format(self.ip_addr, self.name),
311 sudo=True, retry=True)
314 def get_bridges_dc(self):
315 out = self.execute('docker network ls', sudo=True, retry=True)
317 for line in out.splitlines()[1:]:
318 bridges.append(line.split()[1])
321 def get_bridges_brctl(self):
322 out = self.execute('brctl show', retry=True)
324 for line in out.splitlines()[1:]:
325 bridges.append(line.split()[0])
328 def get_bridges_ovs(self):
329 out = self.execute('ovs-vsctl list-br', sudo=True, retry=True)
330 return out.splitlines()
332 def get_bridges(self):
333 if self.br_type == BRIDGE_TYPE_DOCKER:
334 return self.get_bridges_dc()
335 elif self.br_type == BRIDGE_TYPE_BRCTL:
336 return self.get_bridges_brctl()
337 elif self.br_type == BRIDGE_TYPE_OVS:
338 return self.get_bridges_ovs()
341 return self.name in self.get_bridges()
343 def execute(self, cmd, capture=True, sudo=False, retry=False):
349 return m(cmd, capture=capture, try_times=3, interval=1)
351 return m(cmd, capture=capture)
353 def check_br_addr(self, br):
355 cmd = "ip a show dev %s" % br
356 for line in self.execute(cmd, sudo=True).split('\n'):
357 if line.strip().startswith("inet "):
358 elems = [e.strip() for e in line.strip().split(' ')]
360 elif line.strip().startswith("inet6 "):
361 elems = [e.strip() for e in line.strip().split(' ')]
365 def next_ip_address(self):
366 return "{0}/{1}".format(next(self._ip_generator),
367 self.subnet.prefixlen)
369 def addif(self, ctn):
370 name = ctn.next_if_name()
371 self.ctns.append(ctn)
376 ip_address = self.next_ip_address()
377 ip_address_ip = ip_address.split('/')[0]
379 if netaddr.IPNetwork(ip_address).version == 6:
381 opt_ip = "--ip %s" % ip_address_ip
385 opt_ip = "--ip6 %s" % ip_address_ip
387 cmd = "docker network connect %s %s %s" % (
388 opt_ip, self.name, ctn.docker_name())
389 self.execute(cmd, sudo=True)
390 ctn.set_addr_info(bridge=self.name, ipv4=ipv4, ipv6=ipv6,
394 ip_address = self.next_ip_address()
396 if netaddr.IPNetwork(ip_address).version == 6:
398 ctn.pipework(self, ip_address, name, version=version)
400 ctn.pipework(self, '0/0', name)
403 def delete(self, check_exist=True):
407 if self.br_type == BRIDGE_TYPE_DOCKER:
408 self.execute("docker network rm %s" % self.name,
409 sudo=True, retry=True)
410 elif self.br_type == BRIDGE_TYPE_BRCTL:
411 self.execute("ip link set down dev %s" % self.name,
412 sudo=True, retry=True)
414 "ip link delete %s type bridge" % self.name,
415 sudo=True, retry=True)
416 elif self.br_type == BRIDGE_TYPE_OVS:
418 "ovs-vsctl del-br %s" % self.name,
419 sudo=True, retry=True)
422 class Container(object):
423 def __init__(self, name, image=None):
426 self.shared_volumes = []
429 self.is_running = False
436 def docker_name(self):
437 if TEST_PREFIX == DEFAULT_TEST_PREFIX:
439 return '{0}_{1}'.format(TEST_PREFIX, self.name)
441 def get_docker_id(self):
445 return self.docker_name()
447 def next_if_name(self):
448 name = 'eth{0}'.format(len(self.eths) + 1)
449 self.eths.append(name)
452 def set_addr_info(self, bridge, ipv4=None, ipv6=None, ifname='eth0'):
454 self.ip_addrs.append((ifname, ipv4, bridge))
456 self.ip6_addrs.append((ifname, ipv6, bridge))
458 def get_addr_info(self, bridge, ipv=4):
461 ip_addrs = self.ip_addrs
463 ip_addrs = self.ip6_addrs
466 for addr in ip_addrs:
467 if addr[2] == bridge:
468 addrinfo[addr[1]] = addr[0]
471 def execute(self, cmd, capture=True, sudo=False, retry=False):
477 return m(cmd, capture=capture, try_times=3, interval=1)
479 return m(cmd, capture=capture)
481 def dcexec(self, cmd, capture=True, retry=False):
483 return self.cmd.sudo(cmd, capture=capture, try_times=3, interval=1)
485 return self.cmd.sudo(cmd, capture=capture)
487 def exec_on_ctn(self, cmd, capture=True, detach=False):
488 name = self.docker_name()
489 flag = '-d' if detach else ''
490 return self.dcexec('docker exec {0} {1} {2}'.format(
491 flag, name, cmd), capture=capture)
493 def get_containers(self, allctn=False):
494 cmd = 'docker ps --no-trunc=true'
497 out = self.dcexec(cmd, retry=True)
499 for line in out.splitlines()[1:]:
500 containers.append(line.split()[-1])
503 def exist(self, allctn=False):
504 return self.docker_name() in self.get_containers(allctn=allctn)
508 c << "docker run --privileged=true"
509 for sv in self.shared_volumes:
510 c << "-v {0}:{1}".format(sv[0], sv[1])
511 c << "--name {0} --hostname {0} -id {1}".format(self.docker_name(),
513 self.id = self.dcexec(str(c), retry=True)
514 self.is_running = True
515 self.exec_on_ctn("ip li set up dev lo")
518 for line in self.exec_on_ctn("ip a show dev eth0").split('\n'):
519 if line.strip().startswith("inet "):
520 elems = [e.strip() for e in line.strip().split(' ')]
522 elif line.strip().startswith("inet6 "):
523 elems = [e.strip() for e in line.strip().split(' ')]
525 self.set_addr_info(bridge='docker0', ipv4=ipv4, ipv6=ipv6,
529 def stop(self, check_exist=True):
531 if not self.exist(allctn=False):
533 ctn_id = self.get_docker_id()
534 out = self.dcexec('docker stop -t 0 %s' % ctn_id, retry=True)
535 self.is_running = False
538 def remove(self, check_exist=True):
540 if not self.exist(allctn=True):
542 ctn_id = self.get_docker_id()
543 out = self.dcexec('docker rm -f %s' % ctn_id, retry=True)
544 self.is_running = False
547 def pipework(self, bridge, ip_addr, intf_name="", version=4):
548 if not self.is_running:
549 LOG.warning('Call run() before pipeworking')
552 c << "pipework {0}".format(bridge.name)
555 c << "-i {0}".format(intf_name)
565 c << "{0} {1}".format(self.docker_name(), ip_addr)
566 self.set_addr_info(bridge=bridge.name, ipv4=ipv4, ipv6=ipv6,
568 self.execute(str(c), sudo=True, retry=True)
572 cmd = "docker inspect -f '{{.State.Pid}}' %s" % self.docker_name()
573 return int(self.dcexec(cmd))
576 def start_tcpdump(self, interface=None, filename=None):
580 filename = "{0}/{1}.dump".format(
581 self.shared_volumes[0][1], interface)
583 "tcpdump -i {0} -w {1}".format(interface, filename),
587 class BGPContainer(Container):
591 DEFAULT_PEER_ARG = {'neigh_addr': '',
595 'is_rs_client': False,
596 'is_rr_client': False,
602 'graceful_restart': None,
604 'prefix_limit': None}
605 default_peer_keys = sorted(DEFAULT_PEER_ARG.keys())
606 DEFAULT_ROUTE_ARG = {'prefix': None,
614 'extended-community': None,
617 default_route_keys = sorted(DEFAULT_ROUTE_ARG.keys())
619 def __init__(self, name, asn, router_id, ctn_image_name=None):
620 self.config_dir = TEST_BASE_DIR
622 self.config_dir = os.path.join(self.config_dir, TEST_PREFIX)
623 self.config_dir = os.path.join(self.config_dir, name)
625 self.router_id = router_id
629 super(BGPContainer, self).__init__(name, ctn_image_name)
631 'rm -rf {0}'.format(self.config_dir), sudo=True)
632 self.execute('mkdir -p {0}'.format(self.config_dir))
633 self.execute('chmod 777 {0}'.format(self.config_dir))
636 return str({'name': self.name, 'asn': self.asn,
637 'router_id': self.router_id})
639 def run(self, wait=False, w_time=WAIT_FOR_BOOT):
641 super(BGPContainer, self).run()
646 def add_peer(self, peer, bridge='', reload_config=True, v6=False,
648 peer_info = peer_info or {}
649 self.peers[peer] = self.DEFAULT_PEER_ARG.copy()
650 self.peers[peer].update(peer_info)
651 peer_keys = sorted(self.peers[peer].keys())
652 if peer_keys != self.default_peer_keys:
653 raise Exception("argument error peer_info: %s" % peer_info)
657 it = itertools.product(self.ip_addrs, peer.ip_addrs)
659 it = itertools.product(self.ip6_addrs, peer.ip6_addrs)
662 if bridge != '' and bridge != me[2]:
668 addr, mask = local_addr.split('/')
669 local_addr = "{0}%{1}/{2}".format(addr, me[0], mask)
673 raise Exception('peer {0} seems not ip reachable'.format(peer))
675 if not self.peers[peer]['policies']:
676 self.peers[peer]['policies'] = {}
678 self.peers[peer]['neigh_addr'] = neigh_addr
679 self.peers[peer]['local_addr'] = local_addr
680 if self.is_running and reload_config:
684 def del_peer(self, peer, reload_config=True):
686 if self.is_running and reload_config:
690 def disable_peer(self, peer):
691 raise NotImplementedError()
693 def enable_peer(self, peer):
694 raise NotImplementedError()
697 return self.execute('cat {0}/*.log'.format(self.config_dir))
699 def add_route(self, route, reload_config=True, route_info=None):
700 route_info = route_info or {}
701 self.routes[route] = self.DEFAULT_ROUTE_ARG.copy()
702 self.routes[route].update(route_info)
703 route_keys = sorted(self.routes[route].keys())
704 if route_keys != self.default_route_keys:
705 raise Exception("argument error route_info: %s" % route_info)
706 self.routes[route]['prefix'] = route
707 if self.is_running and reload_config:
711 def add_policy(self, policy, peer, typ, default='accept',
713 self.set_default_policy(peer, typ, default)
714 self.define_policy(policy)
715 self.assign_policy(peer, policy, typ)
716 if self.is_running and reload_config:
720 def set_default_policy(self, peer, typ, default):
721 if (typ in ['in', 'out', 'import', 'export'] and
722 default in ['reject', 'accept']):
723 if 'default-policy' not in self.peers[peer]:
724 self.peers[peer]['default-policy'] = {}
725 self.peers[peer]['default-policy'][typ] = default
727 raise Exception('wrong type or default')
729 def define_policy(self, policy):
730 self.policies[policy['name']] = policy
732 def assign_policy(self, peer, policy, typ):
733 if peer not in self.peers:
734 raise Exception('peer {0} not found'.format(peer.name))
735 name = policy['name']
736 if name not in self.policies:
737 raise Exception('policy {0} not found'.format(name))
738 self.peers[peer]['policies'][typ] = policy
740 def get_local_rib(self, peer, rf):
741 raise NotImplementedError()
743 def get_global_rib(self, rf):
744 raise NotImplementedError()
746 def get_neighbor_state(self, peer_id):
747 raise NotImplementedError()
749 def get_reachablily(self, prefix, timeout=20):
750 version = netaddr.IPNetwork(prefix).version
751 addr = prefix.split('/')[0]
758 'unsupported route family: {0}'.format(version))
759 cmd = '/bin/bash -c "/bin/{0} -c 1 -w 1 {1} | xargs echo"'.format(
764 res = self.exec_on_ctn(cmd)
766 if '1 packets received' in res and '0% packet loss':
771 raise Exception('timeout')
774 def wait_for(self, expected_state, peer, timeout=120):
778 state = self.get_neighbor_state(peer)
779 LOG.info("%s's peer %s state: %s",
780 self.router_id, peer.router_id, state)
781 if state == expected_state:
787 raise Exception('timeout')
789 def add_static_route(self, network, next_hop):
790 cmd = '/sbin/ip route add {0} via {1}'.format(network, next_hop)
791 self.exec_on_ctn(cmd)
793 def set_ipv6_forward(self):
794 cmd = 'sysctl -w net.ipv6.conf.all.forwarding=1'
795 self.exec_on_ctn(cmd)
797 def create_config(self):
798 raise NotImplementedError()
800 def reload_config(self):
801 raise NotImplementedError()