From 4ede93ce7bed76b469e1d87447e9a84c33912f43 Mon Sep 17 00:00:00 2001 From: Toby Date: Fri, 16 Feb 2024 19:08:19 -0800 Subject: [PATCH 1/4] fix: incorrect "virgin" handling causing rules with only built-in keywords to fail (#61) --- engine/tcp.go | 20 ++++++++------------ engine/udp.go | 19 ++++++++----------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/engine/tcp.go b/engine/tcp.go index a4744dd..b9fe156 100644 --- a/engine/tcp.go +++ b/engine/tcp.go @@ -60,13 +60,6 @@ func (f *tcpStreamFactory) New(ipFlow, tcpFlow gopacket.Flow, tcp *layers.TCP, a rs := f.Ruleset f.RulesetMutex.RUnlock() ans := analyzersToTCPAnalyzers(rs.Analyzers(info)) - if len(ans) == 0 { - ctx := ac.(*tcpContext) - ctx.Verdict = tcpVerdictAcceptStream - f.Logger.TCPStreamAction(info, ruleset.ActionAllow, true) - // a tcpStream with no activeEntries is a no-op - return &tcpStream{finalVerdict: tcpVerdictAcceptStream} - } // Create entries for each analyzer entries := make([]*tcpStreamEntry, 0, len(ans)) for _, a := range ans { @@ -109,7 +102,7 @@ type tcpStream struct { ruleset ruleset.Ruleset activeEntries []*tcpStreamEntry doneEntries []*tcpStreamEntry - finalVerdict tcpVerdict + lastVerdict tcpVerdict } type tcpStreamEntry struct { @@ -120,11 +113,14 @@ type tcpStreamEntry struct { } func (s *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { - if len(s.activeEntries) > 0 { + if len(s.activeEntries) > 0 || s.virgin { + // Make sure every stream matches against the ruleset at least once, + // even if there are no activeEntries, as the ruleset may have built-in + // properties that need to be matched. return true } else { ctx := ac.(*tcpContext) - ctx.Verdict = s.finalVerdict + ctx.Verdict = s.lastVerdict return false } } @@ -159,7 +155,7 @@ func (s *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass action := result.Action if action != ruleset.ActionMaybe && action != ruleset.ActionModify { verdict := actionToTCPVerdict(action) - s.finalVerdict = verdict + s.lastVerdict = verdict ctx.Verdict = verdict s.logger.TCPStreamAction(s.info, action, false) // Verdict issued, no need to process any more packets @@ -168,7 +164,7 @@ func (s *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } if len(s.activeEntries) == 0 && ctx.Verdict == tcpVerdictAccept { // All entries are done but no verdict issued, accept stream - s.finalVerdict = tcpVerdictAcceptStream + s.lastVerdict = tcpVerdictAcceptStream ctx.Verdict = tcpVerdictAcceptStream s.logger.TCPStreamAction(s.info, ruleset.ActionAllow, true) } diff --git a/engine/udp.go b/engine/udp.go index e97c3ac..89f407b 100644 --- a/engine/udp.go +++ b/engine/udp.go @@ -61,12 +61,6 @@ func (f *udpStreamFactory) New(ipFlow, udpFlow gopacket.Flow, udp *layers.UDP, u rs := f.Ruleset f.RulesetMutex.RUnlock() ans := analyzersToUDPAnalyzers(rs.Analyzers(info)) - if len(ans) == 0 { - uc.Verdict = udpVerdictAcceptStream - f.Logger.UDPStreamAction(info, ruleset.ActionAllow, true) - // a udpStream with no activeEntries is a no-op - return &udpStream{finalVerdict: udpVerdictAcceptStream} - } // Create entries for each analyzer entries := make([]*udpStreamEntry, 0, len(ans)) for _, a := range ans { @@ -167,7 +161,7 @@ type udpStream struct { ruleset ruleset.Ruleset activeEntries []*udpStreamEntry doneEntries []*udpStreamEntry - finalVerdict udpVerdict + lastVerdict udpVerdict } type udpStreamEntry struct { @@ -178,10 +172,13 @@ type udpStreamEntry struct { } func (s *udpStream) Accept(udp *layers.UDP, rev bool, uc *udpContext) bool { - if len(s.activeEntries) > 0 { + if len(s.activeEntries) > 0 || s.virgin { + // Make sure every stream matches against the ruleset at least once, + // even if there are no activeEntries, as the ruleset may have built-in + // properties that need to be matched. return true } else { - uc.Verdict = s.finalVerdict + uc.Verdict = s.lastVerdict return false } } @@ -227,17 +224,17 @@ func (s *udpStream) Feed(udp *layers.UDP, rev bool, uc *udpContext) { } if action != ruleset.ActionMaybe { verdict, final := actionToUDPVerdict(action) + s.lastVerdict = verdict uc.Verdict = verdict s.logger.UDPStreamAction(s.info, action, false) if final { - s.finalVerdict = verdict s.closeActiveEntries() } } } if len(s.activeEntries) == 0 && uc.Verdict == udpVerdictAccept { // All entries are done but no verdict issued, accept stream - s.finalVerdict = udpVerdictAcceptStream + s.lastVerdict = udpVerdictAcceptStream uc.Verdict = udpVerdictAcceptStream s.logger.UDPStreamAction(s.info, ruleset.ActionAllow, true) } From 94cfe7b2c1e8905be343dcfb27b25834faa4db75 Mon Sep 17 00:00:00 2001 From: rootmelo92118 <32770959+rootmelo92118@users.noreply.github.com> Date: Sun, 18 Feb 2024 06:03:50 +0800 Subject: [PATCH 2/4] Add the configuration for the path of geoip and geosite. (#57) * Update README.md * Update README.zh.md * Update README.ja.md * minor tweaks --------- Co-authored-by: Toby --- README.ja.md | 6 ++++++ README.md | 6 ++++++ README.zh.md | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/README.ja.md b/README.ja.md index 018332e..973d635 100644 --- a/README.ja.md +++ b/README.ja.md @@ -75,6 +75,12 @@ workers: tcpMaxBufferedPagesTotal: 4096 tcpMaxBufferedPagesPerConn: 64 udpMaxStreams: 4096 + +# 特定のローカルGeoIP / GeoSiteデータベースファイルを読み込むためのパス。 +# 設定されていない場合は、https://github.com/LoyalSoldier/v2ray-rules-dat から自動的にダウンロードされます。 +# geo: +# geoip: geoip.dat +# geosite: geosite.dat ``` ### ルール例 diff --git a/README.md b/README.md index 4e48017..b41ce72 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,12 @@ workers: tcpMaxBufferedPagesTotal: 4096 tcpMaxBufferedPagesPerConn: 64 udpMaxStreams: 4096 + +# The path to load specific local geoip/geosite db files. +# If not set, they will be automatically downloaded from https://github.com/Loyalsoldier/v2ray-rules-dat +# geo: +# geoip: geoip.dat +# geosite: geosite.dat ``` ### Example rules diff --git a/README.zh.md b/README.zh.md index a466e33..f3efd39 100644 --- a/README.zh.md +++ b/README.zh.md @@ -76,6 +76,12 @@ workers: tcpMaxBufferedPagesTotal: 4096 tcpMaxBufferedPagesPerConn: 64 udpMaxStreams: 4096 + +# 指定的 geoip/geosite 档案路径 +# 如果未设置，将自动从 https://github.com/Loyalsoldier/v2ray-rules-dat 下载 +# geo: +# geoip: geoip.dat +# geosite: geosite.dat ``` ### 样例规则 From ebff4308e4bb8252dcba1e3c41f06e4f177fd8b6 Mon Sep 17 00:00:00 2001 From: Rinka Date: Sun, 18 Feb 2024 06:21:12 +0800 Subject: [PATCH 3/4] Add CIDR support for expr (#62) * feat: add cidr support for expr * docs: add example for cidr * minor code tweaks --------- Co-authored-by: Toby --- README.ja.md | 4 ++++ README.md | 4 ++++ README.zh.md | 4 ++++ ruleset/builtins/cidr.go | 18 ++++++++++++++++ ruleset/expr.go | 45 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 ruleset/builtins/cidr.go diff --git a/README.ja.md b/README.ja.md index 973d635..a8e9d16 100644 --- a/README.ja.md +++ b/README.ja.md @@ -130,6 +130,10 @@ workers: - name: block CN geoip action: block expr: geoip(string(ip.dst), "cn") + +- name: block cidr + action: block + expr: cidr(string(ip.dst), "192.168.0.0/16") ``` #### サポートされるアクション diff --git a/README.md b/README.md index b41ce72..890bcc7 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ to [Expr Language Definition](https://expr-lang.org/docs/language-definition). - name: block CN geoip action: block expr: geoip(string(ip.dst), "cn") + +- name: block cidr + action: block + expr: cidr(string(ip.dst), "192.168.0.0/16") ``` #### Supported actions diff --git a/README.zh.md b/README.zh.md index f3efd39..25dc2e7 100644 --- a/README.zh.md +++ b/README.zh.md @@ -131,6 +131,10 @@ workers: - name: block CN geoip action: block expr: geoip(string(ip.dst), "cn") + +- name: block cidr + action: block + expr: cidr(string(ip.dst), "192.168.0.0/16") ``` #### 支持的 action diff --git a/ruleset/builtins/cidr.go b/ruleset/builtins/cidr.go new file mode 100644 index 0000000..669d469 --- /dev/null +++ b/ruleset/builtins/cidr.go @@ -0,0 +1,18 @@ +package builtins + +import ( + "net" +) + +func MatchCIDR(ip string, cidr *net.IPNet) bool { + ipAddr := net.ParseIP(ip) + if ipAddr == nil { + return false + } + return cidr.Contains(ipAddr) +} + +func CompileCIDR(cidr string) (*net.IPNet, error) { + _, ipNet, err := net.ParseCIDR(cidr) + return ipNet, err +} diff --git a/ruleset/expr.go b/ruleset/expr.go index 738ed7c..4512e1f 100644 --- a/ruleset/expr.go +++ b/ruleset/expr.go @@ -2,6 +2,7 @@ package ruleset import ( "fmt" + "net" "os" "reflect" "strings" @@ -14,6 +15,7 @@ import ( "github.com/apernet/OpenGFW/analyzer" "github.com/apernet/OpenGFW/modifier" + "github.com/apernet/OpenGFW/ruleset/builtins" "github.com/apernet/OpenGFW/ruleset/builtins/geo" ) @@ -100,17 +102,21 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier return nil, fmt.Errorf("rule %q has invalid action %q", rule.Name, rule.Action) } visitor := &idVisitor{Identifiers: make(map[string]bool)} + patcher := &idPatcher{} program, err := expr.Compile(rule.Expr, func(c *conf.Config) { c.Strict = false c.Expect = reflect.Bool - c.Visitors = append(c.Visitors, visitor) + c.Visitors = append(c.Visitors, visitor, patcher) registerBuiltinFunctions(c.Functions, geoMatcher) }, ) if err != nil { return nil, fmt.Errorf("rule %q has invalid expression: %w", rule.Name, err) } + if patcher.Err != nil { + return nil, fmt.Errorf("rule %q failed to patch expression: %w", rule.Name, patcher.Err) + } for name := range visitor.Identifiers { if isBuiltInAnalyzer(name) { continue @@ -126,6 +132,8 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier if err := geoMatcher.LoadGeoSite(); err != nil { return nil, fmt.Errorf("rule %q failed to load geosite: %w", rule.Name, err) } + case "cidr": + // No initialization needed for CIDR. default: a, ok := fullAnMap[name] if !ok { @@ -179,6 +187,13 @@ func registerBuiltinFunctions(funcMap map[string]*ast.Function, geoMatcher *geo. }, Types: []reflect.Type{reflect.TypeOf(geoMatcher.MatchGeoSite)}, } + funcMap["cidr"] = &ast.Function{ + Name: "cidr", + Func: func(params ...any) (any, error) { + return builtins.MatchCIDR(params[0].(string), params[1].(*net.IPNet)), nil + }, + Types: []reflect.Type{reflect.TypeOf((func(string, string) bool)(nil)), reflect.TypeOf(builtins.MatchCIDR)}, + } } func streamInfoToExprEnv(info StreamInfo) map[string]interface{} { @@ -247,6 +262,8 @@ func modifiersToMap(mods []modifier.Modifier) map[string]modifier.Modifier { return modMap } +// idVisitor is a visitor that collects all identifiers in an expression. +// This is for determining which analyzers are used by the expression. type idVisitor struct { Identifiers map[string]bool } @@ -256,3 +273,29 @@ func (v *idVisitor) Visit(node *ast.Node) { v.Identifiers[idNode.Value] = true } } + +// idPatcher patches the AST during expr compilation, replacing certain values with +// their internal representations for better runtime performance. +type idPatcher struct { + Err error +} + +func (p *idPatcher) Visit(node *ast.Node) { + switch (*node).(type) { + case *ast.CallNode: + callNode := (*node).(*ast.CallNode) + switch callNode.Func.Name { + case "cidr": + cidrStringNode, ok := callNode.Arguments[1].(*ast.StringNode) + if !ok { + return + } + cidr, err := builtins.CompileCIDR(cidrStringNode.Value) + if err != nil { + p.Err = err + return + } + callNode.Arguments[1] = &ast.ConstantNode{Value: cidr} + } + } +} From 5c77cede3d2d5e2b51dc6f7c7218fcc37f88cbea Mon Sep 17 00:00:00 2001 From: Toby Date: Sat, 17 Feb 2024 14:28:57 -0800 Subject: [PATCH 4/4] feat: update gopacket to latest master --- go.mod | 2 +- go.sum | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cde5ff7..c28b535 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/coreos/go-iptables v0.7.0 github.com/expr-lang/expr v1.15.7 github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf - github.com/google/gopacket v1.1.19 + github.com/google/gopacket v1.1.20-0.20220810144506-32ee38206866 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mdlayher/netlink v1.6.0 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 9a8cee0..51d4ea0 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/gopacket v1.1.20-0.20220810144506-32ee38206866 h1:NaJi58bCZZh0jjPw78EqDZekPEfhlzYE01C5R+zh1tE= +github.com/google/gopacket v1.1.20-0.20220810144506-32ee38206866/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -72,6 +72,9 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -95,6 +98,8 @@ golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=