1
2
3
4
5
6
7
echo "" > README.md
git add *
git commit -m "init"
[main 836cf99] init
 1 file changed, 1 insertion(+), 92 deletions(-)
 rewrite README.md (99%)
git push

1
2
curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh" | sudo bash
dnf install gitlab-runner -y

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
gitlab-runner register --url http://10.10.10.97/ --registration-token GR1348941ozPELDfBh6ciKzCNajud

Runtime platform                                    arch=amd64 os=linux pid=2675 revision=dcfb4b66 version=15.10.1
Running in system-mode.                            

Enter the GitLab instance URL (for example, https://gitlab.com/):
[http://10.10.10.97/]:
Enter the registration token:
[GR1348941ozPELDfBh6ciKzCNajud]:
Enter a description for the runner:
[runner3.decon230.org]:
Enter tags for the runner (comma-separated):

Enter optional maintenance note for the runner:

Registering runner... succeeded                     runner=GR1348941ozPELDfB
Enter an executor: virtualbox, docker+machine, instance, custom, docker-ssh, shell, docker-ssh+machine, kubernetes, docker, parallels, ssh:
shell
Runner registered successfully.

1
gitlab-runner start

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
dnf module enable python39
alternatives --config python

There are 3 programs which provide 'python'.

  Selection    Command
-----------------------------------------------
*  1           /usr/libexec/no-python
   2           /usr/bin/python3
 + 3           /usr/bin/python3.9

3

pip3 install ansible-core ansible ansible-lint
dnf -y install yamllint
ansible-galaxy collection install pfsensible.core

1
2
3
4
5
6
7
8
9
10
11
12
13
14
git add *
git commit -m "1st checkin"
[main 696846a] 1st checkin
 1 file changed, 24 insertions(+)
 create mode 100644 pb_pfsense_add_aliases.yml
heiko@jellyfish:~/virtenv/repo/pfsense$ git push
Enumerating objects: 4, done.
Counting objects: 100% (4/4), done.
Delta compression using up to 8 threads
Compressing objects: 100% (3/3), done.
Writing objects: 100% (3/3), 629 bytes | 629.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
To 10.10.10.97:root/pfsense.git
   deda056..696846a  main -> main

1
2
3
4
5
6
7
8
ssh-keygen -t ed25519
Generating public/private ed25519 key pair.
Enter file in which to save the key (/home/gitlab-runner/.ssh/id_ed25519):
Created directory '/home/gitlab-runner/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/gitlab-runner/.ssh/id_ed25519.
Your public key has been saved in /home/gitlab-runner/.ssh/id_ed25519.pub.

1
ssh-copy-id ansible@10.10.10.95

1
2
3
4
5
[root@ctlnode ~]# tree ~ansible/
/home/ansible/
|-- README.md
`-- pb_pfsense_add_aliases.yml

1
ansible-galaxy collection install pfsensible.core

1
2
3
4
heiko@jellyfish:~/virtenv$ python -m venv ansi
heiko@jellyfish:~/virtenv$ cd ansi
heiko@jellyfish:~/virtenv/ansi$ source bin/activate
(ansi) heiko@jellyfish:~/virtenv/ansi$

1
2
(ansi) heiko@jellyfish:~/virtenv/ansi$ pip3 install pip-review
(ansi) heiko@jellyfish:~/virtenv/ansi$ pip3 install pip_search

1
(ansi) heiko@jellyfish:~/virtenv/ansi$ pip3 install ansible-core

1
2
3
4
5
6
7
8
9
10
11
(ansi) heiko@jellyfish:~/virtenv/ansi$ ansible-galaxy collection install extreme.exos
(ansi) heiko@jellyfish:~/virtenv/ansi$ ansible-galaxy collection install community.network

(ansi) heiko@jellyfish:~/virtenv/ansi$ ansible-galaxy collection list

Collection        Version
----------------- -------
ansible.netcommon 3.1.3  
ansible.utils     2.6.1  
community.network 4.0.1  
extreme.exos      0.1.1  

1
(ansi) heiko@jellyfish:~/virtenv/ansi$ pip-review -a

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
vyos@mcr1# show interfaces
 ethernet eth0 {
     address 172.23.40.11/24
     description "-> ISP"
     hw-id 00:0c:29:0f:32:0b
 }
 ethernet eth1 {
     address 10.10.10.111/24
     description MGM
     hw-id 00:0c:29:0f:32:15
 }
 ethernet eth2 {
     address 172.23.20.10/24
     description LAN
     hw-id 00:0c:29:0f:32:1f
 }
 loopback lo {
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
vyos@mcr2# show interfaces
 ethernet eth0 {
     address 172.23.40.12/24
     description "-> ISP"
     hw-id 00:0c:29:5a:17:8d
 }
 ethernet eth1 {
     address 10.10.10.112/24
     description MGM
     hw-id 00:0c:29:5a:17:97
 }
 ethernet eth2 {
     address 172.23.30.10/24
     description LAN
     hw-id 00:0c:29:5a:17:a1
 }
 loopback lo {
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
vyos@rp1# show interfaces
 ethernet eth0 {
     address dhcp
     description Internet
     hw-id 00:0c:29:de:43:d5
 }
 ethernet eth1 {
     address 10.10.10.113/24
     description MGM
     hw-id 00:0c:29:de:43:df
 }
 ethernet eth2 {
     address 172.23.40.100/24
     description Transit
     hw-id 00:0c:29:de:43:e9
 }
 loopback lo {
 }

1
2
3
4
5
vyos@rp1# show interfaces loopback
 loopback lo {
     address 192.168.100.1/32
     description Loopback
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
vyos@rp1# sh
 default-information {
     originate {
         ipv4 {
             level-1 {
             }
         }
     }
 }
 interface eth0 {
     passive
 }
 interface eth1 {
     passive
 }
 interface eth2 {
     password {
         plaintext-password 12341234
     }
 }
 interface lo {
     passive
 }
 level level-1
 log-adjacency-changes
 metric-style wide
 net 49.0023.1921.6810.0001.00

1
2
3
4
5
vyos@mcr1# show interfaces loopback
 loopback lo {
     address 192.168.210.1/32
     description Loopback
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
vyos@mcr1# show protocols isis
 interface eth0 {
     password {
         plaintext-password 12341234
     }
     priority 0
 }
 interface eth1 {
     passive
 }
 interface eth2 {
     passive
 }
 interface lo {
     passive
 }
 level level-1
 log-adjacency-changes
 metric-style wide
 net 49.0023.1921.6821.0001.00

1
2
3
4
5
vyos@mcr2# show interfaces loopback
 loopback lo {
     address 192.168.220.1/32
     description Loopback
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
vyos@mcr2# show protocols isis
 interface eth0 {
     password {
         plaintext-password 12341234
     }
     priority 0
 }
 interface eth1 {
     passive
 }
 interface eth2 {
     passive
 }
 interface lo {
     passive
 }
 level level-1
 log-adjacency-changes
 metric-style wide
 net 49.0023.1921.6822.0001.00

1
2
3
4
5
6
7
8
9
10
vyos@mcr1# show protocols pim
 interface eth0 {
 }
 interface eth2 {
 }
 rp {
     address 192.168.255.1 {
         group 224.0.0.0/4
     }
 }

1
2
3
vyos@mcr1# show protocols igmp
 interface eth2 {
 }

1
2
3
4
5
6
7
8
9
10
vyos@mcr1# show protocols pim
 interface eth0 {
 }
 interface eth2 {
 }
 rp {
     address 192.168.255.1 {
         group 224.0.0.0/4
     }
 }

1
2
3
vyos@mcr1# show protocols igmp
 interface eth2 {
 }

1
2
3
4
5
vyos@rp1# show interfaces dummy
 dummy dum0 {
     address 192.168.255.1/32
     description "RP Interface"
 }

1
2
3
4
5
6
7
8
9
10
11
12
vyos@rp1# show protocols pim
 interface dum0 {
 }
 interface eth1 {
 }
 interface eth2 {
 }
 rp {
     address 192.168.255.1 {
         group 224.0.0.0/4
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
vyos@rp1# show policy
 prefix-list dummy-into-isis {
     rule 5 {
         action permit
         prefix 192.168.255.1/32
     }
 }
 route-map dummy-into-isis {
     description "Multicast Interface into ISIS"
     rule 5 {
         action permit
         match {
             ip {
                 address {
                     prefix-list dummy-into-isis
                 }
             }
         }
     }
 }

1
2
3
4
5
6
7
8
vyos@rp1# show protocols isis redistribute
 ipv4 {
     connected {
         level-1 {
             route-map dummy-into-isis
         }
     }
 }

1
2
3
4
5
vyos@mcr1# run show ip route 192.168.255.1
Routing entry for 192.168.255.1/32
  Known via "isis", distance 115, metric 10, best
  Last update 00:04:20 ago
  * 172.23.40.100, via eth0, weight 1

1
root@mc01:/opt/mcjoin/bin# ./mcjoin -s 224.6.7.8

1
root@mc02:/opt/mcjoin/bin# ./mcjoin -s 225.6.7.8

1
2
3
set interfaces loopback lo address 10.2.2.2/32
del protocols static route 10.40.0.0/24
del protocols static route 192.168.0.0/24

1
2
3
4
set interfaces loopback lo address 172.16.0.1/24
set interfaces loopback lo address 172.16.1.1/24
set interfaces loopback lo address 172.16.2.1/24
set interface eth3 address 10.200.40.10/24

1
2
set protocols static route 10.1.1.1/32 interface wg0
set interfaces wireguard wg0 peer WG-R1 allowed-ips 10.1.1.1/32

1
2
set interfaces loopback lo address 10.1.1.1/32
del protocols static route 192.168.1.0/24

1
2
3
set interfaces loopback lo address 172.17.0.1/24
set interfaces loopback lo address 172.17.1.1/24
set interfaces loopback lo address 172.17.2.1/24

1
2
set protocols static route 10.2.2.2/32 interface wg0
set interfaces wireguard wg0 peer WG-R1 allowed-ips 10.2.2.2/32

1
2
3
4
5
6
7
8
vyos@wgr2# show policy community-list
 community-list 1 {
     description "permit AS65100 routing updates"
     rule 1 {
         action permit
         regex 65100:100
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
vyos@wgr2# show policy prefix-list
 prefix-list connected-into-bgp {
     description "connected prefixes into bgp"
     rule 5 {
         action permit
         le 24
         prefix 172.16.0.0/16
     }
     rule 10 {
         action permit
         prefix 10.200.40.0/24
     }
     rule 15 {
         action permit
         prefix 192.168.1.0/24
     }
 }
 prefix-list deny-incoming-routes {
     description "deny default gw"
     rule 5 {
         action deny
         prefix 0.0.0.0/0
     }
     rule 10 {
         action permit
         le 32
         prefix 0.0.0.0/0
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
vyos@wgr2# show policy route-map
 route-map bgp-in {
     description "BGP incoming"
     rule 10 {
         action permit
         match {
             community {
                 community-list 1
             }
             ip {
                 address {
                     prefix-list deny-incoming-routes
                 }
             }
         }
     }
 }
 route-map bgp-out {
     description "BGP outgoing"
     rule 10 {
         action permit
         match {
             ip {
                 address {
                     prefix-list connected-into-bgp
                 }
             }
         }
         set {
             community {
                 replace 65200:100
             }
         }
     }
 }
 route-map connected-into-bgp {
     description "connected into bgp"
     rule 5 {
         action permit
         match {
             ip {
                 address {
                     prefix-list connected-into-bgp
                 }
             }
         }
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
vyos@wgr2# sh protocols bgp
 address-family {
     ipv4-unicast {
         redistribute {
             connected {
                 route-map connected-into-bgp
             }
         }
     }
 }
 neighbor 10.1.1.1 {
     peer-group wireguard
     remote-as 65100
 }
 parameters {
     router-id 10.2.2.2
 }
 peer-group wireguard {
     address-family {
         ipv4-unicast {
             maximum-prefix 10
             route-map {
                 export bgp-out
                 import bgp-in
             }
             soft-reconfiguration {
                 inbound
             }
         }
     }
     description "Wireguard Routers BGP"
     disable-connected-check
     password pass1234
     update-source lo
 }
 system-as 65200

1
2
3
4
5
6
7
8
vyos@wgr1# show policy community-list
 community-list 1 {
     description "permit AS65200 routing updates"
     rule 1 {
         action permit
         regex 65200:100
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
vyos@wgr1# show policy prefix-list
 prefix-list connected-into-bgp {
     description "connected prefixes into bgp"
     rule 5 {
         action permit
         le 24
         prefix 172.17.0.0/16
     }
     rule 10 {
         action permit
         prefix 192.168.0.0/24
     }
     rule 15 {
         action permit
         prefix 10.40.0.0/24
     }
 }
 prefix-list deny-incoming-routes {
     description "filter incoming routes"
     rule 5 {
         action deny
         prefix 0.0.0.0/0
     }
     rule 10 {
         action permit
         le 32
         prefix 0.0.0.0/0
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
vyos@wgr1# sh policy route-map
 route-map bgp-in {
     description "BGP incoming"
     rule 10 {
         action permit
         match {
             community {
                 community-list 1
             }
             ip {
                 address {
                     prefix-list deny-incoming-routes
                 }
             }
         }
     }
 }
 route-map bgp-out {
     description "BGP outgoing"
     rule 10 {
         action permit
         match {
             ip {
                 address {
                     prefix-list connected-into-bgp
                 }
             }
         }
         set {
             community {
                 replace 65100:100
             }
         }
     }
 }
 route-map connected-into-bgp {
     description "connected into bgp"
     rule 5 {
         action permit
         match {
             ip {
                 address {
                     prefix-list connected-into-bgp
                 }
             }
         }
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
vyos@wgr1# sh protocols bgp
 address-family {
     ipv4-unicast {
         redistribute {
             connected {
                 route-map connected-into-bgp
             }
         }
     }
 }
 neighbor 10.2.2.2 {
     peer-group wireguard
     remote-as 65200
 }
 parameters {
     router-id 10.1.1.1
 }
 peer-group wireguard {
     address-family {
         ipv4-unicast {
             maximum-prefix 10
             route-map {
                 export bgp-out
                 import bgp-in
             }
             soft-reconfiguration {
                 inbound
             }
         }
     }
     description "Wireguard Routers BGP"
     disable-connected-check
     password pass1234
     update-source lo
 }
 system-as 65100

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
vyos@wgr1# show interfaces wireguard wg0
 address 10.30.0.1/30
 description VPN->WG-R2
 mtu 1420
 peer WG-R2 {
     address 172.23.30.11
     allowed-ips 10.30.0.0/30
     allowed-ips 192.168.1.0/24
     allowed-ips 10.200.40.0/24
     allowed-ips 172.16.0.0/16
     allowed-ips 10.2.2.2/32
     port 51820
     public-key DOEIvmBXgMOY3va+xCo9ag4F0pzTOC5VsfwuSbESvVM=
 }
 port 51820
 private-key 4IyNUD+pduTl4cdL1EKVs44Cdx3HtdxCFPeomXod2HM=

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
vyos@wgr2# show interfaces wireguard wg0
 address 10.30.0.2/30
 description VPN->WG-R1
 mtu 1420
 peer WG-R1 {
     address 172.23.20.11
     allowed-ips 10.30.0.0/30
     allowed-ips 172.16.0.0/16
     allowed-ips 10.40.0.0/24
     allowed-ips 192.168.0.0/24
     allowed-ips 10.1.1.1/32
     port 51820
     public-key Pz2oyw2876hk5+dOOqwWoN4bFsM9ThR20efHbgqAg1c=
 }
 port 51820
 private-key SOfyUcADKHg/aVBCv7WFJjfDzaIGFRWMSm+TDwNvTGc=      

1
2
3
4
5
6
7
8
9
10
11
12
13
vyos@wgr2:~$ sh ip bgp summary

IPv4 Unicast Summary (VRF default):
BGP router identifier 10.2.2.2, local AS number 65200 vrf-id 0
BGP table version 52
RIB entries 19, using 3648 bytes of memory
Peers 1, using 724 KiB of memory
Peer groups 1, using 64 bytes of memory

Neighbor        V         AS   MsgRcvd   MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd   PfxSnt Desc
10.1.1.1        4      65100       252       257        0    0    0 02:45:35            5        5 N/A

Total number of neighbors 1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
vyos@wgr2:~$ sh ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

S>* 0.0.0.0/0 [1/0] via 172.23.30.10, eth1, weight 1, 03:43:29
S>* 10.1.1.1/32 [1/0] is directly connected, wg0, weight 1, 03:43:29
C>* 10.2.2.2/32 is directly connected, lo, 03:43:32
C>* 10.10.10.0/24 is directly connected, eth0, 03:43:31
C>* 10.30.0.0/30 is directly connected, wg0, 03:43:29
B>  10.40.0.0/24 [20/0] via 10.1.1.1 (recursive), weight 1, 02:40:11
  *                       via 10.1.1.1, wg0 onlink, weight 1, 02:40:11
C>* 10.200.40.0/24 is directly connected, eth3, 03:43:31
C>* 172.16.0.0/24 is directly connected, lo, 03:43:32
C>* 172.16.1.0/24 is directly connected, lo, 03:43:32
C>* 172.16.2.0/24 is directly connected, lo, 03:43:32
B>  172.17.0.0/24 [20/0] via 10.1.1.1 (recursive), weight 1, 02:46:11
  *                        via 10.1.1.1, wg0 onlink, weight 1, 02:46:11
B>  172.17.1.0/24 [20/0] via 10.1.1.1 (recursive), weight 1, 02:46:11
  *                        via 10.1.1.1, wg0 onlink, weight 1, 02:46:11
B>  172.17.2.0/24 [20/0] via 10.1.1.1 (recursive), weight 1, 02:46:11
  *                        via 10.1.1.1, wg0 onlink, weight 1, 02:46:11
C>* 172.23.30.0/24 is directly connected, eth1, 03:43:31
B>  192.168.0.0/24 [20/0] via 10.1.1.1 (recursive), weight 1, 02:46:11
  *                         via 10.1.1.1, wg0 onlink, weight 1, 02:46:11
C>* 192.168.1.0/24 is directly connected, eth2, 03:43:31

1
2
3
4
5
client1:~# traceroute 10.200.40.100
traceroute to 10.200.40.100 (10.200.40.100), 30 hops max, 46 byte packets
 1  10.40.0.1 (10.40.0.1)  1.504 ms  1.890 ms  1.108 ms
 2  10.30.0.2 (10.30.0.2)  3.753 ms  3.686 ms  2.536 ms
 3  10.200.40.100 (10.200.40.100)  3.284 ms  4.484 ms  3.351 ms

1
2
client1:~# curl 10.200.40.100
<html><body><h1>It works!</h1></body></html>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
vyos@wgr1# run generate pki wireguard key-pair install interface wg1

vyos@wgr1# show interfaces wireguard wg1
 address 10.40.0.1/24
 description Remote-VPN
 mtu 1420
 peer RemoteVPN {
     allowed-ips 0.0.0.0/32
     persistent-keepalive 15
     public-key NVWMpPecjOMhczxXOj9+bEiPfgvBBx9b9FRgaRNWf3M=
 }
 port 2224
 private-key 2BL+aYPsxkf89lOnDhJqIhAh/qjhS1I98jn6oyFwkFw=

1
generate wireguard client-config remote1 interface wg1 server 172.23.20.11 address 10.40.0.2/24

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
WireGuard client configuration for interface: wg1

To enable this configuration on a VyOS router you can use the following commands:

=== VyOS (server) configurtation ===

set interfaces wireguard wg1 peer remote1 allowed-ips '10.40.0.2/32'
set interfaces wireguard wg1 peer remote1 public-key 'hF0DUTJd71U8dBn5RiF1RW8LsnH5oHZGbUR+pnmWvXo='

=== RoadWarrior (client) configuration ===


[Interface]
PrivateKey = AC+077T2NP4iJMKzDp70NlOdQzfGIBOBRv2sWt4fW3c=
Address = 10.40.0.2/32
DNS = 1.1.1.1

[Peer]
PublicKey = NVWMpPecjOMhczxXOj9+bEiPfgvBBx9b9FRgaRNWf3M=
Endpoint = 172.23.20.11:2224
AllowedIPs = 0.0.0.0/0, ::/0

1
delete interfaces wireguard wg1 peer RemoteVPN

1
2
3
4
5
6
7
8
9
10
11
yos@wgr1# show interfaces wireguard wg1
 address 10.40.0.1/24
 description Remote-VPN
 mtu 1420
 peer remote1 {
     allowed-ips 10.40.0.2/32
     persistent-keepalive 15
     public-key hF0DUTJd71U8dBn5RiF1RW8LsnH5oHZGbUR+pnmWvXo=
 }
 port 2224
 private-key 2BL+aYPsxkf89lOnDhJqIhAh/qjhS1I98jn6oyFwkFw=

1
2
apk add wireguard-tools
modprobe wireguard

1
cd /etc/wireguard

1
2
3
4
5
6
7
8
9
10
[Interface]
PrivateKey = AC+077T2NP4iJMKzDp70NlOdQzfGIBOBRv2sWt4fW3c=
Address = 10.40.0.2/32
DNS = 1.1.1.1

[Peer]
PublicKey = NVWMpPecjOMhczxXOj9+bEiPfgvBBx9b9FRgaRNWf3M=
Endpoint = 172.23.20.11:2224
AllowedIPs = 0.0.0.0/0, ::/0

1
wg-quick up wg0

1
set interfaces wireguard wg0 peer WG-R1 allowed-ips 10.40.0.0/24

1
set protocols static route 10.40.0.0/24 interface wg0

1
2
3
4
5
yos@wgr2# run ping 10.40.0.2
PING 10.40.0.2 (10.40.0.2) 56(84) bytes of data.
64 bytes from 10.40.0.2: icmp_seq=1 ttl=63 time=2.25 ms
64 bytes from 10.40.0.2: icmp_seq=2 ttl=63 time=3.58 ms
64 bytes from 10.40.0.2: icmp_seq=3 ttl=63 time=3.52 ms

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
vyos@wgr1# show interfaces
 ethernet eth1 {
     address 172.23.20.11/24
     description Uplink->ISP
     hw-id 00:0c:29:47:22:3d
 }
 ethernet eth2 {
     address 192.168.0.10/24
     description LAN
     hw-id 00:0c:29:47:22:47
 }
 loopback lo {
 }

 vyos@wgr1# sh protocols
 static {
     route 0.0.0.0/0 {
         next-hop 172.23.20.10 {
         }
     }
 }

1
2
3
4
5
6
7
8
9
10
11
12
13
vyos@wgr1# show interfaces wireguard wg0
 address 10.30.0.1/30
 description VPN->WG-R2
 mtu 1420
 peer WG-R2 {
     address 172.23.30.11
     allowed-ips 192.168.1.0/24
     allowed-ips 10.30.0.0/30
     port 51820
     public-key DOEIvmBXgMOY3va+xCo9ag4F0pzTOC5VsfwuSbESvVM=
 }
 port 51820
 private-key 4IyNUD+pduTl4cdL1EKVs44Cdx3HtdxCFPeomXod2HM=

1
2
3
4
5
6
7
8
9
10
11
12
13
vyos@wgr2# show interfaces wireguard wg0
 address 10.30.0.2/30
 description VPN->WG-R1
 mtu 1420
 peer WG-R1 {
     address 172.23.20.11
     allowed-ips 192.168.0.0/24
     allowed-ips 10.30.0.0/30
     port 51820
     public-key Pz2oyw2876hk5+dOOqwWoN4bFsM9ThR20efHbgqAg1c=
 }
 port 51820
 private-key SOfyUcADKHg/aVBCv7WFJjfDzaIGFRWMSm+TDwNvTGc=

1
set protocols static route 192.168.1.0/24 interface wg0

1
set protocols static route 192.168.0.0/24 interface wg0

1
2
3
4
5
6
vyos@wgr1:~$ show interfaces wireguard

Codes: S - State, L - Link, u - Up, D - Down, A - Admin Down
Interface        IP Address                        S/L  Description
---------        ----------                        ---  -----------
wg0              10.30.0.1/30                      u/u  VPN->WG-R2

1
2
3
4
5
6
7
8
9
10
vyos@wgr1:~$ show interfaces wireguard wg0 summary

interface: wg0
  public key: Pz2oyw2876hk5+dOOqwWoN4bFsM9ThR20efHbgqAg1c=
  private key: (hidden)
  listening port: 51820

peer: DOEIvmBXgMOY3va+xCo9ag4F0pzTOC5VsfwuSbESvVM=
  endpoint: 172.23.30.11:51820
  allowed ips: 192.168.1.0/24

1
2
3
4
5
6
7
8
9
vyos@wgr1:~$ ping 192.168.1.10
PING 192.168.1.10 (192.168.1.10) 56(84) bytes of data.
64 bytes from 192.168.1.10: icmp_seq=1 ttl=64 time=1.11 ms
64 bytes from 192.168.1.10: icmp_seq=2 ttl=64 time=2.37 ms
64 bytes from 192.168.1.10: icmp_seq=3 ttl=64 time=9.97 ms
64 bytes from 192.168.1.10: icmp_seq=4 ttl=64 time=2.18 ms
^C
--- 192.168.1.10 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3007ms

1
/usr/bin/docker run -i -t -d -p 127.0.0.1:9981:80 -v onlyoffice-fonts:/usr/share/fonts --restart=always --name=office onlyoffice/documentserver

1
/usr/bin/docker run -i -t -d -p 127.0.0.1:8080:8080 --restart=always --name=draw fjudith/draw.io

1
/usr/bin/docker run -i -t -d -p 9000:9000 --restart=always --name=image h2non/imaginary -p 9000 -enable-url-source

1
2
3
4
5
6
fdisk /dev/sda
n p 1 <enter> <enter> t 8e w
pvcreate /dev/sda1
vgextend ol /dev/sda1
lvcreate -n cloud -l +100%FREE ol
mkdir /cloudstore

1
2
dnf install oracle-epel-release-el8
dnf install drbd drbd-bash-completion

1
2
3
4
5
drbdadm create-md cloud
initializing activity log
initializing bitmap (320 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.

1
2
systemctl start drbd
drbdadm up cloud

1
drbdadm primary cloud --force

1
2
3
4
5
6
version: 8.4.11 (api:1/proto:86-101)
srcversion: FC3433D849E3B88C1E7B55C
0: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r-----
ns:1312392 nr:0 dw:0 dr:1315928 al:8 bm:0 lo:0 pe:1 ua:2 ap:0 ep:1 wo:f oos:9169564
[=>..................] sync ed: 12.6% (8952/10232)M
finish: 0:04:06 speed: 37,168 (37,476) K/sec

1
2
mkfs -t ext4 /dev/drbd0
mount /dev/drbd0 /cloudstore/

1
2
3
4
5
6
7
8
9
dnf install pacemaker pcs corosync drbd-pacemaker

systemctl enable --now pcsd.service

usermod -s /bin/bash hacluster

mkdir /home/hacluster

passwd hacluster

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
[root@nc1 ~]# pcs host auth nc1
Username: hacluster
Password:
nc1: Authorized

[root@nc1 ~]# pcs host auth nc2
Username: hacluster
Password:
nc2: Authorized

[root@nc1 ~]# pcs cluster setup cloud nc1 addr=172.23.241.51 nc2 addr=172.23.241.52

Destroying cluster on hosts: 'nc1', 'nc2'...
nc1: Successfully destroyed cluster
nc2: Successfully destroyed cluster
Requesting remove 'pcsd settings' from 'nc1', 'nc2'
nc1: successful removal of the file 'pcsd settings'
nc2: successful removal of the file 'pcsd settings'
Sending 'corosync authkey', 'pacemaker authkey' to 'nc1', 'nc2'
nc1: successful distribution of the file 'corosync authkey'
nc1: successful distribution of the file 'pacemaker authkey'
nc2: successful distribution of the file 'corosync authkey'
nc2: successful distribution of the file 'pacemaker authkey'
Sending 'corosync.conf' to 'nc1', 'nc2'
nc1: successful distribution of the file 'corosync.conf'
nc2: successful distribution of the file 'corosync.conf'
Cluster has been successfully set up.

[root@nc1 ~]# pcs cluster start --all
nc1: Starting Cluster...
nc2: Starting Cluster...

[root@nc1 ~]# pcs cluster enable --all
nc1: Cluster Enabled
nc2: Cluster Enabled

pcs property set stonith-enabled=false
pcs property set no-quorum-policy=ignore

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[root@nc1 ~]# pcs status
Cluster name: cloud
Cluster Summary:
* Stack: corosync
* Current DC: nc2 (version 2.1.2-4.0.1.el8_6.2-ada5c3b36e2) - partition with quorum
* Last updated: Wed Jul 13 12:08:29 2022
* Last change: Wed Jul 13 12:06:55 2022 by root via cibadmin on nc1
* 2 nodes configured
* 0 resource instances configured

Node List:
* Online: [ nc1 nc2 ]

Full List of Resources:
* No resources

Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled

1
2
3
4
5
[root@nc1 ~]# pcs resource create VIP ocf:heartbeat:IPaddr2 ip=172.23.241.50 \
 cidr_netmask=24 op monitor interval=30s

[root@nc1 ~]# pcs resource
* VIP (ocf::heartbeat:IPaddr2): Started nc1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
[root@nc1 ~]# mount | grep -i cloud
/dev/drbd0 on /cloudstore type ext4 (rw,relatime)

[root@nc1 ~]# pcs status
Cluster name: cloud
Cluster Summary:
* Stack: corosync
* Current DC: nc1 (version 2.1.2-4.0.1.el8_6.2-ada5c3b36e2) - partition with quorum
* Last updated: Wed Jul 13 13:44:37 2022
* Last change: Wed Jul 13 13:32:35 2022 by root via cibadmin on nc1
* 2 nodes configured
* 4 resource instances configured

Node List:
* Online: [ nc1 nc2 ]

Full List of Resources:
* VIP (ocf::heartbeat:IPaddr2): Started nc1
* Clone Set: clone_clouddata [clouddata] (promotable):
* Masters: [ nc1 ]
* Stopped: [ nc2 ]
* fs_clouddata (ocf::heartbeat:Filesystem): Started nc1

Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
pcs cluster cib apache.cfg

pcs -f apache.cfg resource create webdata ocf:linbit:drbd drbd_resource=web op monitor interval=30s

pcs -f apache.cfg resource promotable webdata clone_webdata master-ax=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true

pcs -f apache.cfg resource create fs_webdata ocf:heartbeat:Filesystem device="/dev/drbd1" directory="/var/www" fstype="ext4"

pcs -f apache.cfg constraint colocation add clone_webdata with VIP INFINITY

pcs -f apache.cfg constraint order VIP then clone_webdata

pcs -f apache.cfg constraint colocation add fs_webdata with clone_webdata INFINITY

pcs -f apache.cfg constraint order promote clone_webdata then start fs_webdata

pcs -f apache.cfg resource status

pcs cluster cib-push apache.cfg

1
2
[root@nc1 ~]# mount | grep -i www
/dev/drbd1 on /var/www type ext4 (rw,relatime)

1
2
3
4
5
dnf install oracle-epel-release-el8 -y
dnf install httpd mod_qos mod_ssl -y
cd /var/www
chown apache:apache /html
chmod 750 /html

1
2
3
4
5
vi /etc/httpd/conf.d/status.conf
<Location /server-status>
    SetHandler server-status
    Require local
</Location>

1
2
3
4
5
6
7
8
9
10
11
pcs cluster cib httpd.cfg

pcs -f httpd.cfg resource create httpd ocf:heartbeat:apache configfile=/etc/httpd/conf/httpd.conf statusurl=http://127.0.0.1/server-status

pcs -f httpd.cfg constraint colocation add httpd with VIP INFINITY

pcs -f httpd.cfg constraint order fs_webdata then start httpd

pcs -f httpd.cfg constraint order VIP then httpd

pcs cluster cib-push httpd.cfg

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[root@nc1 ~]# pcs status
Cluster name: cloud
Cluster Summary:
* Stack: corosync
* Current DC: nc1 (version 2.1.2-4.0.1.el8_6.2-ada5c3b36e2) - partition with quorum
* Last updated: Wed Jul 13 14:51:47 2022
* Last change: Wed Jul 13 14:50:47 2022 by root via cibadmin on nc1
* 2 nodes configured
* 8 resource instances configured

Node List:
* Online: [ nc1 nc2 ]

Full List of Resources:
* VIP (ocf::heartbeat:IPaddr2): Started nc1
* Clone Set: clone_clouddata [clouddata] (promotable):
* Masters: [ nc1 ]
* Stopped: [ nc2 ]
* fs_clouddata (ocf::heartbeat:Filesystem): Started nc1
* Clone Set: clone_webdata [webdata] (promotable):
* Masters: [ nc1 ]
* Stopped: [ nc2 ]
* fs_webdata (ocf::heartbeat:Filesystem): Started nc1
* httpd (ocf::heartbeat:apache): Started nc1

Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
dnf install https://rpms.remirepo.net/enterprise/remi-release-8.rpm
dnf module reset php -y
dnf module enable php:remi-8.0

dnf install -y php php-cli php-common php-pdo php-fpm php-mbstring php-opcache php-sodium php-xml \
php-mbstring php-intl php-pecl-zip php-pecl-apcu php-gmp php-bcmath php-ldap php-pecl-xmlrpc \
php-process php-soap php-tidy php-pecl-imagick php-mysqlnd php-pear php-fedora-autoloader \
php-pecl-redis5 php-pecl-igbinary php-pecl-msgpack php-pecl-mcrypt php-gd
/etc/php.ini anpassen:
output_buffering = Off
max_execution_time = 300
max_input_time = 300
memory_limit = 4096M
post_max_size = 1024M
upload_max_filesize = 1024M
max_file_uploads = 100
file_uploads = On

1
2
3
4
5
6
7
8
9
10
11
pcs cluster cib phpfpm.cfg

pcs -f phpfpm.cfg resource create phpfpm systemd:php-fpm

pcs -f phpfpm.cfg constraint colocation add phpfpm with VIP INFINITY

pcs -f phpfpm.cfg constraint order httpd then start phpfpm

pcs -f phpfpm.cfg constraint order VIP then phpfpm

pcs cluster cib-push phpfpm.cfg

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
[root@nc1 ~]# pcs status
Cluster name: cloud
Cluster Summary:
* Stack: corosync
* Current DC: nc1 (version 2.1.2-4.0.1.el8_6.2-ada5c3b36e2) - partition with quorum
* Last updated: Fri Jul 15 17:28:39 2022
* Last change: Fri Jul 15 12:18:49 2022 by root via cibadmin on nc1
* 2 nodes configured
* 9 resource instances configured

Node List:
* Online: [ nc1 nc2 ]

Full List of Resources:
* VIP (ocf::heartbeat:IPaddr2): Started nc1
* Clone Set: clone_clouddata [clouddata] (promotable):
* Masters: [ nc1 ]
* Stopped: [ nc2 ]
* fs_clouddata (ocf::heartbeat:Filesystem): Started nc1
* Clone Set: clone_webdata [webdata] (promotable):
* Masters: [ nc1 ]
* Stopped: [ nc2 ]
* fs_webdata (ocf::heartbeat:Filesystem): Started nc1
* httpd (ocf::heartbeat:apache): Started nc1
* phpfpm (systemd:php-fpm): Started nc1

Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled

1
2
3
4
5
6
7
wget https://download.nextcloud.com/server/releases/latest.zip
unzip latest.zip
cp -rv nextcloud/* /var/www/html
cd /var/www/html
chown -R apache:apache *
find . -type f -print0 | xargs -0 chmod 640
find . -type d -print0 | xargs -0 chmod 750

1
2
cd /var/www/html
sudo -u apache php occ app:install deck

1
crontab -u apache -e

1
2
3
mkdir /opt/cron
vi /opt/cron/nccron
*/15 * * * * apache /bin/php -f /var/www/html/cron.php

1
2
3
4
5
6
7
8
9
10
11
12
pcs cluster cib cron.cfg

pcs -f cron.cfg resource create cron-nextcloud ocf:heartbeat:symlink \
link=/etc/cron.d/nccron target=/opt/cron/nccron op monitor interval=30s

pcs -f cron.cfg constraint colocation add cron-nextcloud with VIP INFINITY

pcs -f cron.cfg constraint order httpd then start cron-nextcloud

pcs -f cron.cfg constraint order VIP then cron-nextcloud

pcs cluster cib-push cron.cfg