From d3f1785ac9e9ca70f10693beabdb542b4f1c52d6 Mon Sep 17 00:00:00 2001 From: Keith Petkus Date: Thu, 8 Aug 2024 13:24:49 -0400 Subject: [PATCH 1/4] feat: netlink queueNum/table config options --- cmd/root.go | 4 ++++ io/nfqueue.go | 56 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 1ccf025..93a4791 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -174,6 +174,8 @@ type cliConfig struct { type cliConfigIO struct { QueueSize uint32 `mapstructure:"queueSize"` + QueueNum uint16 `mapstructure:"queueNum"` + Table string `mapstructure:"table"` ReadBuffer int `mapstructure:"rcvBuf"` WriteBuffer int `mapstructure:"sndBuf"` Local bool `mapstructure:"local"` @@ -221,6 +223,8 @@ func (c *cliConfig) fillIO(config *engine.Config) error { WriteBuffer: c.IO.WriteBuffer, Local: c.IO.Local, RST: c.IO.RST, + QueueNum: c.IO.QueueNum, + Table: c.IO.Table, }) } diff --git a/io/nfqueue.go b/io/nfqueue.go index f1a64df..eeca6d7 100644 --- a/io/nfqueue.go +++ b/io/nfqueue.go @@ -19,18 +19,18 @@ import ( ) const ( - nfqueueNum = 100 + nfqueueDefaultQueueNum = 100 nfqueueMaxPacketLen = 0xFFFF nfqueueDefaultQueueSize = 128 nfqueueConnMarkAccept = 1001 nfqueueConnMarkDrop = 1002 - nftFamily = "inet" - nftTable = "opengfw" + nftFamily = "inet" + nftDefaultTable = "opengfw" ) -func generateNftRules(local, rst bool) (*nftTableSpec, error) { +func generateNftRules(local, rst bool, nfqueueNum int, nftTable string) (*nftTableSpec, error) { if local && rst { return nil, errors.New("tcp rst is not supported in local mode") } @@ -64,7 +64,7 @@ func generateNftRules(local, rst bool) (*nftTableSpec, error) { return table, nil } -func generateIptRules(local, rst bool) ([]iptRule, error) { +func generateIptRules(local, rst bool, nfqueueNum int) ([]iptRule, error) { if local && rst { return nil, errors.New("tcp rst is not supported in local mode") } @@ -94,10 +94,12 @@ var _ PacketIO = (*nfqueuePacketIO)(nil) var errNotNFQueuePacket = errors.New("not an NFQueue packet") type nfqueuePacketIO struct { - n *nfqueue.Nfqueue - local bool - rst bool - rSet bool // whether the nftables/iptables rules have been set + n *nfqueue.Nfqueue + local bool + rst bool + rSet bool // whether the nftables/iptables rules have been set + queueNum int + table string // nftable name // iptables not nil = use iptables instead of nftables ipt4 *iptables.IPTables @@ -108,6 +110,8 @@ type nfqueuePacketIO struct { type NFQueuePacketIOConfig struct { QueueSize uint32 + QueueNum uint16 + Table string ReadBuffer int WriteBuffer int Local bool @@ -118,6 +122,12 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { if config.QueueSize == 0 { config.QueueSize = nfqueueDefaultQueueSize } + if config.QueueNum == 0 { + config.QueueNum = nfqueueDefaultQueueNum + } + if config.Table == "" { + config.Table = nftDefaultTable + } var ipt4, ipt6 *iptables.IPTables var err error if nftCheck() != nil { @@ -132,7 +142,7 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { } } n, err := nfqueue.Open(&nfqueue.Config{ - NfQueue: nfqueueNum, + NfQueue: config.QueueNum, MaxPacketLen: nfqueueMaxPacketLen, MaxQueueLen: config.QueueSize, Copymode: nfqueue.NfQnlCopyPacket, @@ -156,11 +166,13 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { } } return &nfqueuePacketIO{ - n: n, - local: config.Local, - rst: config.RST, - ipt4: ipt4, - ipt6: ipt6, + n: n, + local: config.Local, + rst: config.RST, + queueNum: int(config.QueueNum), + table: config.Table, + ipt4: ipt4, + ipt6: ipt6, protectedDialer: &net.Dialer{ Control: func(network, address string, c syscall.RawConn) error { var err error @@ -214,7 +226,7 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error if n.ipt4 != nil { err = n.setupIpt(n.local, n.rst, false) } else { - err = n.setupNft(n.local, n.rst, false) + err = n.setupNft(n.local, n.rst, false, n.queueNum) } if err != nil { return err @@ -274,7 +286,7 @@ func (n *nfqueuePacketIO) Close() error { if n.ipt4 != nil { _ = n.setupIpt(n.local, n.rst, true) } else { - _ = n.setupNft(n.local, n.rst, true) + _ = n.setupNft(n.local, n.rst, true, n.queueNum) } n.rSet = false } @@ -286,17 +298,17 @@ func (n *nfqueuePacketIO) SetCancelFunc(cancelFunc context.CancelFunc) error { return nil } -func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error { - rules, err := generateNftRules(local, rst) +func (n *nfqueuePacketIO) setupNft(local, rst, remove bool, nfqueueNum int) error { + rules, err := generateNftRules(local, rst, nfqueueNum, n.table) if err != nil { return err } rulesText := rules.String() if remove { - err = nftDelete(nftFamily, nftTable) + err = nftDelete(nftFamily, n.table) } else { // Delete first to make sure no leftover rules - _ = nftDelete(nftFamily, nftTable) + _ = nftDelete(nftFamily, n.table) err = nftAdd(rulesText) } if err != nil { @@ -306,7 +318,7 @@ func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error { } func (n *nfqueuePacketIO) setupIpt(local, rst, remove bool) error { - rules, err := generateIptRules(local, rst) + rules, err := generateIptRules(local, rst, n.queueNum) if err != nil { return err } From d8d7c5b477e710ba5cb28377c2f5db311f824409 Mon Sep 17 00:00:00 2001 From: Haruue Date: Sun, 27 Oct 2024 15:44:04 +0900 Subject: [PATCH 2/4] chore: allow set nfqueue num to 0 --- cmd/root.go | 14 +++++++------- io/nfqueue.go | 11 ++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 93a4791..fc09a96 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -173,13 +173,13 @@ type cliConfig struct { } type cliConfigIO struct { - QueueSize uint32 `mapstructure:"queueSize"` - QueueNum uint16 `mapstructure:"queueNum"` - Table string `mapstructure:"table"` - ReadBuffer int `mapstructure:"rcvBuf"` - WriteBuffer int `mapstructure:"sndBuf"` - Local bool `mapstructure:"local"` - RST bool `mapstructure:"rst"` + QueueSize uint32 `mapstructure:"queueSize"` + QueueNum *uint16 `mapstructure:"queueNum"` + Table string `mapstructure:"table"` + ReadBuffer int `mapstructure:"rcvBuf"` + WriteBuffer int `mapstructure:"sndBuf"` + Local bool `mapstructure:"local"` + RST bool `mapstructure:"rst"` } type cliConfigReplay struct { diff --git a/io/nfqueue.go b/io/nfqueue.go index eeca6d7..58c85c9 100644 --- a/io/nfqueue.go +++ b/io/nfqueue.go @@ -110,7 +110,7 @@ type nfqueuePacketIO struct { type NFQueuePacketIOConfig struct { QueueSize uint32 - QueueNum uint16 + QueueNum *uint16 Table string ReadBuffer int WriteBuffer int @@ -122,8 +122,9 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { if config.QueueSize == 0 { config.QueueSize = nfqueueDefaultQueueSize } - if config.QueueNum == 0 { - config.QueueNum = nfqueueDefaultQueueNum + if config.QueueNum == nil { + queueNum := uint16(nfqueueDefaultQueueNum) + config.QueueNum = &queueNum } if config.Table == "" { config.Table = nftDefaultTable @@ -142,7 +143,7 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { } } n, err := nfqueue.Open(&nfqueue.Config{ - NfQueue: config.QueueNum, + NfQueue: *config.QueueNum, MaxPacketLen: nfqueueMaxPacketLen, MaxQueueLen: config.QueueSize, Copymode: nfqueue.NfQnlCopyPacket, @@ -169,7 +170,7 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { n: n, local: config.Local, rst: config.RST, - queueNum: int(config.QueueNum), + queueNum: int(*config.QueueNum), table: config.Table, ipt4: ipt4, ipt6: ipt6, From 5f4df7e806fc453dbc3d9a3a81e6f4487ed46e49 Mon Sep 17 00:00:00 2001 From: Haruue Date: Sun, 27 Oct 2024 15:44:13 +0900 Subject: [PATCH 3/4] chore: rm nfqueueNum parameter in setupNft() --- io/nfqueue.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/io/nfqueue.go b/io/nfqueue.go index 58c85c9..402be06 100644 --- a/io/nfqueue.go +++ b/io/nfqueue.go @@ -227,7 +227,7 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error if n.ipt4 != nil { err = n.setupIpt(n.local, n.rst, false) } else { - err = n.setupNft(n.local, n.rst, false, n.queueNum) + err = n.setupNft(n.local, n.rst, false) } if err != nil { return err @@ -287,7 +287,7 @@ func (n *nfqueuePacketIO) Close() error { if n.ipt4 != nil { _ = n.setupIpt(n.local, n.rst, true) } else { - _ = n.setupNft(n.local, n.rst, true, n.queueNum) + _ = n.setupNft(n.local, n.rst, true) } n.rSet = false } @@ -299,8 +299,8 @@ func (n *nfqueuePacketIO) SetCancelFunc(cancelFunc context.CancelFunc) error { return nil } -func (n *nfqueuePacketIO) setupNft(local, rst, remove bool, nfqueueNum int) error { - rules, err := generateNftRules(local, rst, nfqueueNum, n.table) +func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error { + rules, err := generateNftRules(local, rst, n.queueNum, n.table) if err != nil { return err } From 0e97c9f0864274bf82406bb5277f72ed3fc23ad8 Mon Sep 17 00:00:00 2001 From: Haruue Date: Mon, 28 Oct 2024 10:17:46 +0900 Subject: [PATCH 4/4] feat: connmark accept/drop config options --- cmd/root.go | 26 +++++++----- io/nfqueue.go | 111 +++++++++++++++++++++++++++++--------------------- 2 files changed, 81 insertions(+), 56 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index fc09a96..1513cca 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -173,13 +173,16 @@ type cliConfig struct { } type cliConfigIO struct { - QueueSize uint32 `mapstructure:"queueSize"` - QueueNum *uint16 `mapstructure:"queueNum"` - Table string `mapstructure:"table"` - ReadBuffer int `mapstructure:"rcvBuf"` - WriteBuffer int `mapstructure:"sndBuf"` - Local bool `mapstructure:"local"` - RST bool `mapstructure:"rst"` + QueueSize uint32 `mapstructure:"queueSize"` + QueueNum *uint16 `mapstructure:"queueNum"` + Table string `mapstructure:"table"` + ConnMarkAccept uint32 `mapstructure:"connMarkAccept"` + ConnMarkDrop uint32 `mapstructure:"connMarkDrop"` + + ReadBuffer int `mapstructure:"rcvBuf"` + WriteBuffer int `mapstructure:"sndBuf"` + Local bool `mapstructure:"local"` + RST bool `mapstructure:"rst"` } type cliConfigReplay struct { @@ -218,13 +221,16 @@ func (c *cliConfig) fillIO(config *engine.Config) error { } else { // Setup IO for nfqueue ioImpl, err = io.NewNFQueuePacketIO(io.NFQueuePacketIOConfig{ - QueueSize: c.IO.QueueSize, + QueueSize: c.IO.QueueSize, + QueueNum: c.IO.QueueNum, + Table: c.IO.Table, + ConnMarkAccept: c.IO.ConnMarkAccept, + ConnMarkDrop: c.IO.ConnMarkDrop, + ReadBuffer: c.IO.ReadBuffer, WriteBuffer: c.IO.WriteBuffer, Local: c.IO.Local, RST: c.IO.RST, - QueueNum: c.IO.QueueNum, - Table: c.IO.Table, }) } diff --git a/io/nfqueue.go b/io/nfqueue.go index 402be06..20f0c11 100644 --- a/io/nfqueue.go +++ b/io/nfqueue.go @@ -23,25 +23,24 @@ const ( nfqueueMaxPacketLen = 0xFFFF nfqueueDefaultQueueSize = 128 - nfqueueConnMarkAccept = 1001 - nfqueueConnMarkDrop = 1002 + nfqueueDefaultConnMarkAccept = 1001 nftFamily = "inet" nftDefaultTable = "opengfw" ) -func generateNftRules(local, rst bool, nfqueueNum int, nftTable string) (*nftTableSpec, error) { - if local && rst { +func (n *nfqueuePacketIO) generateNftRules() (*nftTableSpec, error) { + if n.local && n.rst { return nil, errors.New("tcp rst is not supported in local mode") } table := &nftTableSpec{ Family: nftFamily, - Table: nftTable, + Table: n.table, } - table.Defines = append(table.Defines, fmt.Sprintf("define ACCEPT_CTMARK=%d", nfqueueConnMarkAccept)) - table.Defines = append(table.Defines, fmt.Sprintf("define DROP_CTMARK=%d", nfqueueConnMarkDrop)) - table.Defines = append(table.Defines, fmt.Sprintf("define QUEUE_NUM=%d", nfqueueNum)) - if local { + table.Defines = append(table.Defines, fmt.Sprintf("define ACCEPT_CTMARK=%d", n.connMarkAccept)) + table.Defines = append(table.Defines, fmt.Sprintf("define DROP_CTMARK=%d", n.connMarkDrop)) + table.Defines = append(table.Defines, fmt.Sprintf("define QUEUE_NUM=%d", n.queueNum)) + if n.local { table.Chains = []nftChainSpec{ {Chain: "INPUT", Header: "type filter hook input priority filter; policy accept;"}, {Chain: "OUTPUT", Header: "type filter hook output priority filter; policy accept;"}, @@ -55,7 +54,7 @@ func generateNftRules(local, rst bool, nfqueueNum int, nftTable string) (*nftTab c := &table.Chains[i] c.Rules = append(c.Rules, "meta mark $ACCEPT_CTMARK ct mark set $ACCEPT_CTMARK") // Bypass protected connections c.Rules = append(c.Rules, "ct mark $ACCEPT_CTMARK counter accept") - if rst { + if n.rst { c.Rules = append(c.Rules, "ip protocol tcp ct mark $DROP_CTMARK counter reject with tcp reset") } c.Rules = append(c.Rules, "ct mark $DROP_CTMARK counter drop") @@ -64,12 +63,12 @@ func generateNftRules(local, rst bool, nfqueueNum int, nftTable string) (*nftTab return table, nil } -func generateIptRules(local, rst bool, nfqueueNum int) ([]iptRule, error) { - if local && rst { +func (n *nfqueuePacketIO) generateIptRules() ([]iptRule, error) { + if n.local && n.rst { return nil, errors.New("tcp rst is not supported in local mode") } var chains []string - if local { + if n.local { chains = []string{"INPUT", "OUTPUT"} } else { chains = []string{"FORWARD"} @@ -77,13 +76,13 @@ func generateIptRules(local, rst bool, nfqueueNum int) ([]iptRule, error) { rules := make([]iptRule, 0, 4*len(chains)) for _, chain := range chains { // Bypass protected connections - rules = append(rules, iptRule{"filter", chain, []string{"-m", "mark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "CONNMARK", "--set-mark", strconv.Itoa(nfqueueConnMarkAccept)}}) - rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "ACCEPT"}}) - if rst { - rules = append(rules, iptRule{"filter", chain, []string{"-p", "tcp", "-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "REJECT", "--reject-with", "tcp-reset"}}) + rules = append(rules, iptRule{"filter", chain, []string{"-m", "mark", "--mark", strconv.Itoa(n.connMarkAccept), "-j", "CONNMARK", "--set-mark", strconv.Itoa(n.connMarkAccept)}}) + rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(n.connMarkAccept), "-j", "ACCEPT"}}) + if n.rst { + rules = append(rules, iptRule{"filter", chain, []string{"-p", "tcp", "-m", "connmark", "--mark", strconv.Itoa(n.connMarkDrop), "-j", "REJECT", "--reject-with", "tcp-reset"}}) } - rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "DROP"}}) - rules = append(rules, iptRule{"filter", chain, []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(nfqueueNum), "--queue-bypass"}}) + rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(n.connMarkDrop), "-j", "DROP"}}) + rules = append(rules, iptRule{"filter", chain, []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(n.queueNum), "--queue-bypass"}}) } return rules, nil @@ -94,12 +93,14 @@ var _ PacketIO = (*nfqueuePacketIO)(nil) var errNotNFQueuePacket = errors.New("not an NFQueue packet") type nfqueuePacketIO struct { - n *nfqueue.Nfqueue - local bool - rst bool - rSet bool // whether the nftables/iptables rules have been set - queueNum int - table string // nftable name + n *nfqueue.Nfqueue + local bool + rst bool + rSet bool // whether the nftables/iptables rules have been set + queueNum int + table string // nftable name + connMarkAccept int + connMarkDrop int // iptables not nil = use iptables instead of nftables ipt4 *iptables.IPTables @@ -109,9 +110,12 @@ type nfqueuePacketIO struct { } type NFQueuePacketIOConfig struct { - QueueSize uint32 - QueueNum *uint16 - Table string + QueueSize uint32 + QueueNum *uint16 + Table string + ConnMarkAccept uint32 + ConnMarkDrop uint32 + ReadBuffer int WriteBuffer int Local bool @@ -129,6 +133,19 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { if config.Table == "" { config.Table = nftDefaultTable } + if config.ConnMarkAccept == 0 { + config.ConnMarkAccept = nfqueueDefaultConnMarkAccept + } + if config.ConnMarkDrop == 0 { + config.ConnMarkDrop = config.ConnMarkAccept + 1 + if config.ConnMarkDrop == 0 { + // Overflow + config.ConnMarkDrop = 1 + } + } + if config.ConnMarkAccept == config.ConnMarkDrop { + return nil, errors.New("connMarkAccept and connMarkDrop cannot be the same") + } var ipt4, ipt6 *iptables.IPTables var err error if nftCheck() != nil { @@ -167,18 +184,20 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) { } } return &nfqueuePacketIO{ - n: n, - local: config.Local, - rst: config.RST, - queueNum: int(*config.QueueNum), - table: config.Table, - ipt4: ipt4, - ipt6: ipt6, + n: n, + local: config.Local, + rst: config.RST, + queueNum: int(*config.QueueNum), + table: config.Table, + connMarkAccept: int(config.ConnMarkAccept), + connMarkDrop: int(config.ConnMarkDrop), + ipt4: ipt4, + ipt6: ipt6, protectedDialer: &net.Dialer{ Control: func(network, address string, c syscall.RawConn) error { var err error cErr := c.Control(func(fd uintptr) { - err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, nfqueueConnMarkAccept) + err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, int(config.ConnMarkAccept)) }) if cErr != nil { return cErr @@ -225,9 +244,9 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error } if !n.rSet { if n.ipt4 != nil { - err = n.setupIpt(n.local, n.rst, false) + err = n.setupIpt(false) } else { - err = n.setupNft(n.local, n.rst, false) + err = n.setupNft(false) } if err != nil { return err @@ -267,11 +286,11 @@ func (n *nfqueuePacketIO) SetVerdict(p Packet, v Verdict, newPacket []byte) erro case VerdictAcceptModify: return n.n.SetVerdictModPacket(nP.id, nfqueue.NfAccept, newPacket) case VerdictAcceptStream: - return n.n.SetVerdictWithConnMark(nP.id, nfqueue.NfAccept, nfqueueConnMarkAccept) + return n.n.SetVerdictWithConnMark(nP.id, nfqueue.NfAccept, n.connMarkAccept) case VerdictDrop: return n.n.SetVerdict(nP.id, nfqueue.NfDrop) case VerdictDropStream: - return n.n.SetVerdictWithConnMark(nP.id, nfqueue.NfDrop, nfqueueConnMarkDrop) + return n.n.SetVerdictWithConnMark(nP.id, nfqueue.NfDrop, n.connMarkDrop) default: // Invalid verdict, ignore for now return nil @@ -285,9 +304,9 @@ func (n *nfqueuePacketIO) ProtectedDialContext(ctx context.Context, network, add func (n *nfqueuePacketIO) Close() error { if n.rSet { if n.ipt4 != nil { - _ = n.setupIpt(n.local, n.rst, true) + _ = n.setupIpt(true) } else { - _ = n.setupNft(n.local, n.rst, true) + _ = n.setupNft(true) } n.rSet = false } @@ -299,8 +318,8 @@ func (n *nfqueuePacketIO) SetCancelFunc(cancelFunc context.CancelFunc) error { return nil } -func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error { - rules, err := generateNftRules(local, rst, n.queueNum, n.table) +func (n *nfqueuePacketIO) setupNft(remove bool) error { + rules, err := n.generateNftRules() if err != nil { return err } @@ -318,8 +337,8 @@ func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error { return nil } -func (n *nfqueuePacketIO) setupIpt(local, rst, remove bool) error { - rules, err := generateIptRules(local, rst, n.queueNum) +func (n *nfqueuePacketIO) setupIpt(remove bool) error { + rules, err := n.generateIptRules() if err != nil { return err }