From f07a38bc47a8bb406b45140a26243cb87d9b6a24 Mon Sep 17 00:00:00 2001 From: Rinka Date: Wed, 31 Jan 2024 09:30:35 +0800 Subject: [PATCH] Add GeoIP and GeoSite support for expr (#38) * feat: copy something from hysteria/extras/outbounds/acl * feat: add geoip and geosite support for expr * refactor: geo matcher * fix: typo * refactor: geo matcher * feat: expose config options to specify local geoip/geosite db files * refactor: engine.Config should not contains geo * feat: make geosite and geoip lazy downloaded * chore: minor code improvement * docs: add geoip/geosite usage --------- Co-authored-by: Toby --- README.ja.md | 8 + README.md | 8 + README.zh.md | 8 + cmd/root.go | 12 +- go.mod | 4 + go.sum | 7 + ruleset/builtins/geo/geo_loader.go | 128 ++++ ruleset/builtins/geo/geo_matcher.go | 115 ++++ ruleset/builtins/geo/interface.go | 27 + ruleset/builtins/geo/matchers_v2geo.go | 213 +++++++ ruleset/builtins/geo/v2geo/load.go | 44 ++ ruleset/builtins/geo/v2geo/load_test.go | 54 ++ ruleset/builtins/geo/v2geo/v2geo.pb.go | 745 ++++++++++++++++++++++++ ruleset/builtins/geo/v2geo/v2geo.proto | 76 +++ ruleset/expr.go | 55 +- ruleset/interface.go | 5 + 16 files changed, 1502 insertions(+), 7 deletions(-) create mode 100644 ruleset/builtins/geo/geo_loader.go create mode 100644 ruleset/builtins/geo/geo_matcher.go create mode 100644 ruleset/builtins/geo/interface.go create mode 100644 ruleset/builtins/geo/matchers_v2geo.go create mode 100644 ruleset/builtins/geo/v2geo/load.go create mode 100644 ruleset/builtins/geo/v2geo/load_test.go create mode 100644 ruleset/builtins/geo/v2geo/v2geo.pb.go create mode 100644 ruleset/builtins/geo/v2geo/v2geo.proto diff --git a/README.ja.md b/README.ja.md index f9fced3..fa3fb5f 100644 --- a/README.ja.md +++ b/README.ja.md @@ -103,6 +103,14 @@ workers: - name: block google socks action: block expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80 + +- name: block bilibili geosite + action: block + expr: geosite(string(tls?.req?.sni), "bilibili") + +- name: block CN geoip + action: block + expr: geoip(string(ip.dst), "cn") ``` #### サポートされるアクション diff --git a/README.md b/README.md index 79e431e..6127b71 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,14 @@ to [Expr Language Definition](https://expr-lang.org/docs/language-definition). - name: block google socks action: block expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80 + +- name: block bilibili geosite + action: block + expr: geosite(string(tls?.req?.sni), "bilibili") + +- name: block CN geoip + action: block + expr: geoip(string(ip.dst), "cn") ``` #### Supported actions diff --git a/README.zh.md b/README.zh.md index 8ad4f90..0fb87a1 100644 --- a/README.zh.md +++ b/README.zh.md @@ -102,6 +102,14 @@ workers: - name: block google socks action: block expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80 + +- name: block bilibili geosite + action: block + expr: geosite(string(tls?.req?.sni), "bilibili") + +- name: block CN geoip + action: block + expr: geoip(string(ip.dst), "cn") ``` #### 支持的 action diff --git a/cmd/root.go b/cmd/root.go index 2b21bce..a4bf13d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -161,6 +161,7 @@ func initLogger() { type cliConfig struct { IO cliConfigIO `mapstructure:"io"` Workers cliConfigWorkers `mapstructure:"workers"` + Ruleset cliConfigRuleset `mapstructure:"ruleset"` } type cliConfigIO struct { @@ -176,6 +177,11 @@ type cliConfigWorkers struct { UDPMaxStreams int `mapstructure:"udpMaxStreams"` } +type cliConfigRuleset struct { + GeoIp string `mapstructure:"geoip"` + GeoSite string `mapstructure:"geosite"` +} + func (c *cliConfig) fillLogger(config *engine.Config) error { config.Logger = &engineLogger{} return nil @@ -244,7 +250,11 @@ func runMain(cmd *cobra.Command, args []string) { if err != nil { logger.Fatal("failed to load rules", zap.Error(err)) } - rs, err := ruleset.CompileExprRules(rawRs, analyzers, modifiers) + rsConfig := &ruleset.BuiltinConfig{ + GeoSiteFilename: config.Ruleset.GeoSite, + GeoIpFilename: config.Ruleset.GeoIp, + } + rs, err := ruleset.CompileExprRules(rawRs, analyzers, modifiers, rsConfig) if err != nil { logger.Fatal("failed to compile rules", zap.Error(err)) } diff --git a/go.mod b/go.mod index 605d503..cde5ff7 100644 --- a/go.mod +++ b/go.mod @@ -12,11 +12,14 @@ require ( github.com/mdlayher/netlink v1.6.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -26,6 +29,7 @@ require ( github.com/mdlayher/socket v0.1.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect diff --git a/go.sum b/go.sum index 21e9167..9a8cee0 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/expr-lang/expr v1.15.7 h1:BK0JcWUkoW6nrbLBo6xCKhz4BvH5DSOOu1Gx5lucyZo= github.com/expr-lang/expr v1.15.7/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf h1:NqGS3vTHzVENbIfd87cXZwdpO6MB2R1PjHMJLi4Z3ow= @@ -13,6 +14,8 @@ github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf/go.mod h1:eS github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= @@ -41,6 +44,7 @@ github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -111,6 +115,9 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/ruleset/builtins/geo/geo_loader.go b/ruleset/builtins/geo/geo_loader.go new file mode 100644 index 0000000..de5166a --- /dev/null +++ b/ruleset/builtins/geo/geo_loader.go @@ -0,0 +1,128 @@ +package geo + +import ( + "io" + "net/http" + "os" + "time" + + "github.com/apernet/OpenGFW/ruleset/builtins/geo/v2geo" +) + +const ( + geoipFilename = "geoip.dat" + geoipURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat" + geositeFilename = "geosite.dat" + geositeURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat" + + geoDefaultUpdateInterval = 7 * 24 * time.Hour // 7 days +) + +var _ GeoLoader = (*V2GeoLoader)(nil) + +// V2GeoLoader provides the on-demand GeoIP/MatchGeoSite database +// loading functionality required by the ACL engine. +// Empty filenames = automatic download from built-in URLs. +type V2GeoLoader struct { + GeoIPFilename string + GeoSiteFilename string + UpdateInterval time.Duration + + DownloadFunc func(filename, url string) + DownloadErrFunc func(err error) + + geoipMap map[string]*v2geo.GeoIP + geositeMap map[string]*v2geo.GeoSite +} + +func NewDefaultGeoLoader(geoSiteFilename, geoIpFilename string) *V2GeoLoader { + return &V2GeoLoader{ + GeoIPFilename: geoIpFilename, + GeoSiteFilename: geoSiteFilename, + DownloadFunc: func(filename, url string) {}, + DownloadErrFunc: func(err error) {}, + } +} + +func (l *V2GeoLoader) shouldDownload(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return true + } + dt := time.Now().Sub(info.ModTime()) + if l.UpdateInterval == 0 { + return dt > geoDefaultUpdateInterval + } else { + return dt > l.UpdateInterval + } +} + +func (l *V2GeoLoader) download(filename, url string) error { + l.DownloadFunc(filename, url) + + resp, err := http.Get(url) + if err != nil { + l.DownloadErrFunc(err) + return err + } + defer resp.Body.Close() + + f, err := os.Create(filename) + if err != nil { + l.DownloadErrFunc(err) + return err + } + defer f.Close() + + _, err = io.Copy(f, resp.Body) + l.DownloadErrFunc(err) + return err +} + +func (l *V2GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) { + if l.geoipMap != nil { + return l.geoipMap, nil + } + autoDL := false + filename := l.GeoIPFilename + if filename == "" { + autoDL = true + filename = geoipFilename + } + if autoDL && l.shouldDownload(filename) { + err := l.download(filename, geoipURL) + if err != nil { + return nil, err + } + } + m, err := v2geo.LoadGeoIP(filename) + if err != nil { + return nil, err + } + l.geoipMap = m + return m, nil +} + +func (l *V2GeoLoader) LoadGeoSite() (map[string]*v2geo.GeoSite, error) { + if l.geositeMap != nil { + return l.geositeMap, nil + } + autoDL := false + filename := l.GeoSiteFilename + if filename == "" { + autoDL = true + filename = geositeFilename + } + if autoDL && l.shouldDownload(filename) { + err := l.download(filename, geositeURL) + if err != nil { + return nil, err + } + } + m, err := v2geo.LoadGeoSite(filename) + if err != nil { + return nil, err + } + l.geositeMap = m + return m, nil +} diff --git a/ruleset/builtins/geo/geo_matcher.go b/ruleset/builtins/geo/geo_matcher.go new file mode 100644 index 0000000..2032f90 --- /dev/null +++ b/ruleset/builtins/geo/geo_matcher.go @@ -0,0 +1,115 @@ +package geo + +import ( + "net" + "strings" + "sync" +) + +type GeoMatcher struct { + geoLoader GeoLoader + geoSiteMatcher map[string]hostMatcher + siteMatcherLock sync.Mutex + geoIpMatcher map[string]hostMatcher + ipMatcherLock sync.Mutex +} + +func NewGeoMatcher(geoSiteFilename, geoIpFilename string) (*GeoMatcher, error) { + geoLoader := NewDefaultGeoLoader(geoSiteFilename, geoIpFilename) + + return &GeoMatcher{ + geoLoader: geoLoader, + geoSiteMatcher: make(map[string]hostMatcher), + geoIpMatcher: make(map[string]hostMatcher), + }, nil +} + +func (g *GeoMatcher) MatchGeoIp(ip, condition string) bool { + g.ipMatcherLock.Lock() + defer g.ipMatcherLock.Unlock() + + matcher, ok := g.geoIpMatcher[condition] + if !ok { + // GeoIP matcher + condition = strings.ToLower(condition) + country := condition + if len(country) == 0 { + return false + } + gMap, err := g.geoLoader.LoadGeoIP() + if err != nil { + return false + } + list, ok := gMap[country] + if !ok || list == nil { + return false + } + matcher, err = newGeoIPMatcher(list) + if err != nil { + return false + } + g.geoIpMatcher[condition] = matcher + } + parseIp := net.ParseIP(ip) + if parseIp == nil { + return false + } + ipv4 := parseIp.To4() + if ipv4 != nil { + return matcher.Match(HostInfo{IPv4: ipv4}) + } + ipv6 := parseIp.To16() + if ipv6 != nil { + return matcher.Match(HostInfo{IPv6: ipv6}) + } + return false +} + +func (g *GeoMatcher) MatchGeoSite(site, condition string) bool { + g.siteMatcherLock.Lock() + defer g.siteMatcherLock.Unlock() + + matcher, ok := g.geoSiteMatcher[condition] + if !ok { + // MatchGeoSite matcher + condition = strings.ToLower(condition) + name, attrs := parseGeoSiteName(condition) + if len(name) == 0 { + return false + } + gMap, err := g.geoLoader.LoadGeoSite() + if err != nil { + return false + } + list, ok := gMap[name] + if !ok || list == nil { + return false + } + matcher, err = newGeositeMatcher(list, attrs) + if err != nil { + return false + } + g.geoSiteMatcher[condition] = matcher + } + return matcher.Match(HostInfo{Name: site}) +} + +func (g *GeoMatcher) LoadGeoSite() error { + _, err := g.geoLoader.LoadGeoSite() + return err +} + +func (g *GeoMatcher) LoadGeoIP() error { + _, err := g.geoLoader.LoadGeoIP() + return err +} + +func parseGeoSiteName(s string) (string, []string) { + parts := strings.Split(s, "@") + base := strings.TrimSpace(parts[0]) + attrs := parts[1:] + for i := range attrs { + attrs[i] = strings.TrimSpace(attrs[i]) + } + return base, attrs +} diff --git a/ruleset/builtins/geo/interface.go b/ruleset/builtins/geo/interface.go new file mode 100644 index 0000000..a655539 --- /dev/null +++ b/ruleset/builtins/geo/interface.go @@ -0,0 +1,27 @@ +package geo + +import ( + "fmt" + "net" + + "github.com/apernet/OpenGFW/ruleset/builtins/geo/v2geo" +) + +type HostInfo struct { + Name string + IPv4 net.IP + IPv6 net.IP +} + +func (h HostInfo) String() string { + return fmt.Sprintf("%s|%s|%s", h.Name, h.IPv4, h.IPv6) +} + +type GeoLoader interface { + LoadGeoIP() (map[string]*v2geo.GeoIP, error) + LoadGeoSite() (map[string]*v2geo.GeoSite, error) +} + +type hostMatcher interface { + Match(HostInfo) bool +} diff --git a/ruleset/builtins/geo/matchers_v2geo.go b/ruleset/builtins/geo/matchers_v2geo.go new file mode 100644 index 0000000..0271f33 --- /dev/null +++ b/ruleset/builtins/geo/matchers_v2geo.go @@ -0,0 +1,213 @@ +package geo + +import ( + "bytes" + "errors" + "net" + "regexp" + "sort" + "strings" + + "github.com/apernet/OpenGFW/ruleset/builtins/geo/v2geo" +) + +var _ hostMatcher = (*geoipMatcher)(nil) + +type geoipMatcher struct { + N4 []*net.IPNet // sorted + N6 []*net.IPNet // sorted + Inverse bool +} + +// matchIP tries to match the given IP address with the corresponding IPNets. +// Note that this function does NOT handle the Inverse flag. +func (m *geoipMatcher) matchIP(ip net.IP) bool { + var n []*net.IPNet + if ip4 := ip.To4(); ip4 != nil { + // N4 stores IPv4 addresses in 4-byte form. + // Make sure we use it here too, otherwise bytes.Compare will fail. + ip = ip4 + n = m.N4 + } else { + n = m.N6 + } + left, right := 0, len(n)-1 + for left <= right { + mid := (left + right) / 2 + if n[mid].Contains(ip) { + return true + } else if bytes.Compare(n[mid].IP, ip) < 0 { + left = mid + 1 + } else { + right = mid - 1 + } + } + return false +} + +func (m *geoipMatcher) Match(host HostInfo) bool { + if host.IPv4 != nil { + if m.matchIP(host.IPv4) { + return !m.Inverse + } + } + if host.IPv6 != nil { + if m.matchIP(host.IPv6) { + return !m.Inverse + } + } + return m.Inverse +} + +func newGeoIPMatcher(list *v2geo.GeoIP) (*geoipMatcher, error) { + n4 := make([]*net.IPNet, 0) + n6 := make([]*net.IPNet, 0) + for _, cidr := range list.Cidr { + if len(cidr.Ip) == 4 { + // IPv4 + n4 = append(n4, &net.IPNet{ + IP: cidr.Ip, + Mask: net.CIDRMask(int(cidr.Prefix), 32), + }) + } else if len(cidr.Ip) == 16 { + // IPv6 + n6 = append(n6, &net.IPNet{ + IP: cidr.Ip, + Mask: net.CIDRMask(int(cidr.Prefix), 128), + }) + } else { + return nil, errors.New("invalid IP length") + } + } + // Sort the IPNets, so we can do binary search later. + sort.Slice(n4, func(i, j int) bool { + return bytes.Compare(n4[i].IP, n4[j].IP) < 0 + }) + sort.Slice(n6, func(i, j int) bool { + return bytes.Compare(n6[i].IP, n6[j].IP) < 0 + }) + return &geoipMatcher{ + N4: n4, + N6: n6, + Inverse: list.InverseMatch, + }, nil +} + +var _ hostMatcher = (*geositeMatcher)(nil) + +type geositeDomainType int + +const ( + geositeDomainPlain geositeDomainType = iota + geositeDomainRegex + geositeDomainRoot + geositeDomainFull +) + +type geositeDomain struct { + Type geositeDomainType + Value string + Regex *regexp.Regexp + Attrs map[string]bool +} + +type geositeMatcher struct { + Domains []geositeDomain + // Attributes are matched using "and" logic - if you have multiple attributes here, + // a domain must have all of those attributes to be considered a match. + Attrs []string +} + +func (m *geositeMatcher) matchDomain(domain geositeDomain, host HostInfo) bool { + // Match attributes first + if len(m.Attrs) > 0 { + if len(domain.Attrs) == 0 { + return false + } + for _, attr := range m.Attrs { + if !domain.Attrs[attr] { + return false + } + } + } + + switch domain.Type { + case geositeDomainPlain: + return strings.Contains(host.Name, domain.Value) + case geositeDomainRegex: + if domain.Regex != nil { + return domain.Regex.MatchString(host.Name) + } + case geositeDomainFull: + return host.Name == domain.Value + case geositeDomainRoot: + if host.Name == domain.Value { + return true + } + return strings.HasSuffix(host.Name, "."+domain.Value) + default: + return false + } + return false +} + +func (m *geositeMatcher) Match(host HostInfo) bool { + for _, domain := range m.Domains { + if m.matchDomain(domain, host) { + return true + } + } + return false +} + +func newGeositeMatcher(list *v2geo.GeoSite, attrs []string) (*geositeMatcher, error) { + domains := make([]geositeDomain, len(list.Domain)) + for i, domain := range list.Domain { + switch domain.Type { + case v2geo.Domain_Plain: + domains[i] = geositeDomain{ + Type: geositeDomainPlain, + Value: domain.Value, + Attrs: domainAttributeToMap(domain.Attribute), + } + case v2geo.Domain_Regex: + regex, err := regexp.Compile(domain.Value) + if err != nil { + return nil, err + } + domains[i] = geositeDomain{ + Type: geositeDomainRegex, + Regex: regex, + Attrs: domainAttributeToMap(domain.Attribute), + } + case v2geo.Domain_Full: + domains[i] = geositeDomain{ + Type: geositeDomainFull, + Value: domain.Value, + Attrs: domainAttributeToMap(domain.Attribute), + } + case v2geo.Domain_RootDomain: + domains[i] = geositeDomain{ + Type: geositeDomainRoot, + Value: domain.Value, + Attrs: domainAttributeToMap(domain.Attribute), + } + default: + return nil, errors.New("unsupported domain type") + } + } + return &geositeMatcher{ + Domains: domains, + Attrs: attrs, + }, nil +} + +func domainAttributeToMap(attrs []*v2geo.Domain_Attribute) map[string]bool { + m := make(map[string]bool) + for _, attr := range attrs { + // Supposedly there are also int attributes, + // but nobody seems to use them, so we treat everything as boolean for now. + m[attr.Key] = true + } + return m +} diff --git a/ruleset/builtins/geo/v2geo/load.go b/ruleset/builtins/geo/v2geo/load.go new file mode 100644 index 0000000..2dd918c --- /dev/null +++ b/ruleset/builtins/geo/v2geo/load.go @@ -0,0 +1,44 @@ +package v2geo + +import ( + "os" + "strings" + + "google.golang.org/protobuf/proto" +) + +// LoadGeoIP loads a GeoIP data file and converts it to a map. +// The keys of the map (country codes) are all normalized to lowercase. +func LoadGeoIP(filename string) (map[string]*GeoIP, error) { + bs, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + var list GeoIPList + if err := proto.Unmarshal(bs, &list); err != nil { + return nil, err + } + m := make(map[string]*GeoIP) + for _, entry := range list.Entry { + m[strings.ToLower(entry.CountryCode)] = entry + } + return m, nil +} + +// LoadGeoSite loads a GeoSite data file and converts it to a map. +// The keys of the map (site keys) are all normalized to lowercase. +func LoadGeoSite(filename string) (map[string]*GeoSite, error) { + bs, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + var list GeoSiteList + if err := proto.Unmarshal(bs, &list); err != nil { + return nil, err + } + m := make(map[string]*GeoSite) + for _, entry := range list.Entry { + m[strings.ToLower(entry.CountryCode)] = entry + } + return m, nil +} diff --git a/ruleset/builtins/geo/v2geo/load_test.go b/ruleset/builtins/geo/v2geo/load_test.go new file mode 100644 index 0000000..e9c901a --- /dev/null +++ b/ruleset/builtins/geo/v2geo/load_test.go @@ -0,0 +1,54 @@ +package v2geo + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadGeoIP(t *testing.T) { + m, err := LoadGeoIP("geoip.dat") + assert.NoError(t, err) + + // Exact checks since we know the data. + assert.Len(t, m, 252) + assert.Equal(t, m["cn"].CountryCode, "CN") + assert.Len(t, m["cn"].Cidr, 10407) + assert.Equal(t, m["us"].CountryCode, "US") + assert.Len(t, m["us"].Cidr, 193171) + assert.Equal(t, m["private"].CountryCode, "PRIVATE") + assert.Len(t, m["private"].Cidr, 18) + assert.Contains(t, m["private"].Cidr, &CIDR{ + Ip: []byte("\xc0\xa8\x00\x00"), + Prefix: 16, + }) +} + +func TestLoadGeoSite(t *testing.T) { + m, err := LoadGeoSite("geosite.dat") + assert.NoError(t, err) + + // Exact checks since we know the data. + assert.Len(t, m, 1204) + assert.Equal(t, m["netflix"].CountryCode, "NETFLIX") + assert.Len(t, m["netflix"].Domain, 25) + assert.Contains(t, m["netflix"].Domain, &Domain{ + Type: Domain_Full, + Value: "netflix.com.edgesuite.net", + }) + assert.Contains(t, m["netflix"].Domain, &Domain{ + Type: Domain_RootDomain, + Value: "fast.com", + }) + assert.Len(t, m["google"].Domain, 1066) + assert.Contains(t, m["google"].Domain, &Domain{ + Type: Domain_RootDomain, + Value: "ggpht.cn", + Attribute: []*Domain_Attribute{ + { + Key: "cn", + TypedValue: &Domain_Attribute_BoolValue{BoolValue: true}, + }, + }, + }) +} diff --git a/ruleset/builtins/geo/v2geo/v2geo.pb.go b/ruleset/builtins/geo/v2geo/v2geo.pb.go new file mode 100644 index 0000000..49f0963 --- /dev/null +++ b/ruleset/builtins/geo/v2geo/v2geo.pb.go @@ -0,0 +1,745 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: v2geo.proto + +package v2geo + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type of domain value. +type Domain_Type int32 + +const ( + // The value is used as is. + Domain_Plain Domain_Type = 0 + // The value is used as a regular expression. + Domain_Regex Domain_Type = 1 + // The value is a root domain. + Domain_RootDomain Domain_Type = 2 + // The value is a domain. + Domain_Full Domain_Type = 3 +) + +// Enum value maps for Domain_Type. +var ( + Domain_Type_name = map[int32]string{ + 0: "Plain", + 1: "Regex", + 2: "RootDomain", + 3: "Full", + } + Domain_Type_value = map[string]int32{ + "Plain": 0, + "Regex": 1, + "RootDomain": 2, + "Full": 3, + } +) + +func (x Domain_Type) Enum() *Domain_Type { + p := new(Domain_Type) + *p = x + return p +} + +func (x Domain_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Domain_Type) Descriptor() protoreflect.EnumDescriptor { + return file_v2geo_proto_enumTypes[0].Descriptor() +} + +func (Domain_Type) Type() protoreflect.EnumType { + return &file_v2geo_proto_enumTypes[0] +} + +func (x Domain_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Domain_Type.Descriptor instead. +func (Domain_Type) EnumDescriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{0, 0} +} + +// Domain for routing decision. +type Domain struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Domain matching type. + Type Domain_Type `protobuf:"varint,1,opt,name=type,proto3,enum=Domain_Type" json:"type,omitempty"` + // Domain value. + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Attributes of this domain. May be used for filtering. + Attribute []*Domain_Attribute `protobuf:"bytes,3,rep,name=attribute,proto3" json:"attribute,omitempty"` +} + +func (x *Domain) Reset() { + *x = Domain{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Domain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Domain) ProtoMessage() {} + +func (x *Domain) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Domain.ProtoReflect.Descriptor instead. +func (*Domain) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{0} +} + +func (x *Domain) GetType() Domain_Type { + if x != nil { + return x.Type + } + return Domain_Plain +} + +func (x *Domain) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *Domain) GetAttribute() []*Domain_Attribute { + if x != nil { + return x.Attribute + } + return nil +} + +// IP for routing decision, in CIDR form. +type CIDR struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // IP address, should be either 4 or 16 bytes. + Ip []byte `protobuf:"bytes,1,opt,name=ip,proto3" json:"ip,omitempty"` + // Number of leading ones in the network mask. + Prefix uint32 `protobuf:"varint,2,opt,name=prefix,proto3" json:"prefix,omitempty"` +} + +func (x *CIDR) Reset() { + *x = CIDR{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CIDR) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CIDR) ProtoMessage() {} + +func (x *CIDR) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CIDR.ProtoReflect.Descriptor instead. +func (*CIDR) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{1} +} + +func (x *CIDR) GetIp() []byte { + if x != nil { + return x.Ip + } + return nil +} + +func (x *CIDR) GetPrefix() uint32 { + if x != nil { + return x.Prefix + } + return 0 +} + +type GeoIP struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CountryCode string `protobuf:"bytes,1,opt,name=country_code,json=countryCode,proto3" json:"country_code,omitempty"` + Cidr []*CIDR `protobuf:"bytes,2,rep,name=cidr,proto3" json:"cidr,omitempty"` + InverseMatch bool `protobuf:"varint,3,opt,name=inverse_match,json=inverseMatch,proto3" json:"inverse_match,omitempty"` + // resource_hash instruct simplified config converter to load domain from geo file. + ResourceHash []byte `protobuf:"bytes,4,opt,name=resource_hash,json=resourceHash,proto3" json:"resource_hash,omitempty"` + Code string `protobuf:"bytes,5,opt,name=code,proto3" json:"code,omitempty"` +} + +func (x *GeoIP) Reset() { + *x = GeoIP{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeoIP) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeoIP) ProtoMessage() {} + +func (x *GeoIP) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GeoIP.ProtoReflect.Descriptor instead. +func (*GeoIP) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{2} +} + +func (x *GeoIP) GetCountryCode() string { + if x != nil { + return x.CountryCode + } + return "" +} + +func (x *GeoIP) GetCidr() []*CIDR { + if x != nil { + return x.Cidr + } + return nil +} + +func (x *GeoIP) GetInverseMatch() bool { + if x != nil { + return x.InverseMatch + } + return false +} + +func (x *GeoIP) GetResourceHash() []byte { + if x != nil { + return x.ResourceHash + } + return nil +} + +func (x *GeoIP) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +type GeoIPList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Entry []*GeoIP `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` +} + +func (x *GeoIPList) Reset() { + *x = GeoIPList{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeoIPList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeoIPList) ProtoMessage() {} + +func (x *GeoIPList) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GeoIPList.ProtoReflect.Descriptor instead. +func (*GeoIPList) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{3} +} + +func (x *GeoIPList) GetEntry() []*GeoIP { + if x != nil { + return x.Entry + } + return nil +} + +type GeoSite struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CountryCode string `protobuf:"bytes,1,opt,name=country_code,json=countryCode,proto3" json:"country_code,omitempty"` + Domain []*Domain `protobuf:"bytes,2,rep,name=domain,proto3" json:"domain,omitempty"` + // resource_hash instruct simplified config converter to load domain from geo file. + ResourceHash []byte `protobuf:"bytes,3,opt,name=resource_hash,json=resourceHash,proto3" json:"resource_hash,omitempty"` + Code string `protobuf:"bytes,4,opt,name=code,proto3" json:"code,omitempty"` +} + +func (x *GeoSite) Reset() { + *x = GeoSite{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeoSite) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeoSite) ProtoMessage() {} + +func (x *GeoSite) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MatchGeoSite.ProtoReflect.Descriptor instead. +func (*GeoSite) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{4} +} + +func (x *GeoSite) GetCountryCode() string { + if x != nil { + return x.CountryCode + } + return "" +} + +func (x *GeoSite) GetDomain() []*Domain { + if x != nil { + return x.Domain + } + return nil +} + +func (x *GeoSite) GetResourceHash() []byte { + if x != nil { + return x.ResourceHash + } + return nil +} + +func (x *GeoSite) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +type GeoSiteList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Entry []*GeoSite `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` +} + +func (x *GeoSiteList) Reset() { + *x = GeoSiteList{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeoSiteList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeoSiteList) ProtoMessage() {} + +func (x *GeoSiteList) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GeoSiteList.ProtoReflect.Descriptor instead. +func (*GeoSiteList) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{5} +} + +func (x *GeoSiteList) GetEntry() []*GeoSite { + if x != nil { + return x.Entry + } + return nil +} + +type Domain_Attribute struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Types that are assignable to TypedValue: + // + // *Domain_Attribute_BoolValue + // *Domain_Attribute_IntValue + TypedValue isDomain_Attribute_TypedValue `protobuf_oneof:"typed_value"` +} + +func (x *Domain_Attribute) Reset() { + *x = Domain_Attribute{} + if protoimpl.UnsafeEnabled { + mi := &file_v2geo_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Domain_Attribute) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Domain_Attribute) ProtoMessage() {} + +func (x *Domain_Attribute) ProtoReflect() protoreflect.Message { + mi := &file_v2geo_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Domain_Attribute.ProtoReflect.Descriptor instead. +func (*Domain_Attribute) Descriptor() ([]byte, []int) { + return file_v2geo_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *Domain_Attribute) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (m *Domain_Attribute) GetTypedValue() isDomain_Attribute_TypedValue { + if m != nil { + return m.TypedValue + } + return nil +} + +func (x *Domain_Attribute) GetBoolValue() bool { + if x, ok := x.GetTypedValue().(*Domain_Attribute_BoolValue); ok { + return x.BoolValue + } + return false +} + +func (x *Domain_Attribute) GetIntValue() int64 { + if x, ok := x.GetTypedValue().(*Domain_Attribute_IntValue); ok { + return x.IntValue + } + return 0 +} + +type isDomain_Attribute_TypedValue interface { + isDomain_Attribute_TypedValue() +} + +type Domain_Attribute_BoolValue struct { + BoolValue bool `protobuf:"varint,2,opt,name=bool_value,json=boolValue,proto3,oneof"` +} + +type Domain_Attribute_IntValue struct { + IntValue int64 `protobuf:"varint,3,opt,name=int_value,json=intValue,proto3,oneof"` +} + +func (*Domain_Attribute_BoolValue) isDomain_Attribute_TypedValue() {} + +func (*Domain_Attribute_IntValue) isDomain_Attribute_TypedValue() {} + +var File_v2geo_proto protoreflect.FileDescriptor + +var file_v2geo_proto_rawDesc = []byte{ + 0x0a, 0x0b, 0x76, 0x32, 0x67, 0x65, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x02, + 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0c, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x2f, 0x0a, 0x09, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x41, 0x74, 0x74, + 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x52, 0x09, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, + 0x65, 0x1a, 0x6c, 0x0a, 0x09, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x42, 0x0d, 0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, + 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x6c, 0x61, 0x69, 0x6e, + 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x67, 0x65, 0x78, 0x10, 0x01, 0x12, 0x0e, 0x0a, + 0x0a, 0x52, 0x6f, 0x6f, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x10, 0x02, 0x12, 0x08, 0x0a, + 0x04, 0x46, 0x75, 0x6c, 0x6c, 0x10, 0x03, 0x22, 0x2e, 0x0a, 0x04, 0x43, 0x49, 0x44, 0x52, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x70, 0x12, + 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0xa3, 0x01, 0x0a, 0x05, 0x47, 0x65, 0x6f, 0x49, + 0x50, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, + 0x43, 0x6f, 0x64, 0x65, 0x12, 0x19, 0x0a, 0x04, 0x63, 0x69, 0x64, 0x72, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x05, 0x2e, 0x43, 0x49, 0x44, 0x52, 0x52, 0x04, 0x63, 0x69, 0x64, 0x72, 0x12, + 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x76, 0x65, 0x72, 0x73, 0x65, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x29, 0x0a, + 0x09, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x05, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x06, 0x2e, 0x47, 0x65, 0x6f, 0x49, + 0x50, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x86, 0x01, 0x0a, 0x07, 0x47, 0x65, 0x6f, + 0x53, 0x69, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, + 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x72, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0c, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, + 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x22, 0x2d, 0x0a, 0x0b, 0x47, 0x65, 0x6f, 0x53, 0x69, 0x74, 0x65, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x1e, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x08, 0x2e, 0x47, 0x65, 0x6f, 0x53, 0x69, 0x74, 0x65, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x76, 0x32, 0x67, 0x65, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_v2geo_proto_rawDescOnce sync.Once + file_v2geo_proto_rawDescData = file_v2geo_proto_rawDesc +) + +func file_v2geo_proto_rawDescGZIP() []byte { + file_v2geo_proto_rawDescOnce.Do(func() { + file_v2geo_proto_rawDescData = protoimpl.X.CompressGZIP(file_v2geo_proto_rawDescData) + }) + return file_v2geo_proto_rawDescData +} + +var file_v2geo_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_v2geo_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_v2geo_proto_goTypes = []interface{}{ + (Domain_Type)(0), // 0: Domain.Type + (*Domain)(nil), // 1: Domain + (*CIDR)(nil), // 2: CIDR + (*GeoIP)(nil), // 3: GeoIP + (*GeoIPList)(nil), // 4: GeoIPList + (*GeoSite)(nil), // 5: MatchGeoSite + (*GeoSiteList)(nil), // 6: GeoSiteList + (*Domain_Attribute)(nil), // 7: Domain.Attribute +} +var file_v2geo_proto_depIdxs = []int32{ + 0, // 0: Domain.type:type_name -> Domain.Type + 7, // 1: Domain.attribute:type_name -> Domain.Attribute + 2, // 2: GeoIP.cidr:type_name -> CIDR + 3, // 3: GeoIPList.entry:type_name -> GeoIP + 1, // 4: MatchGeoSite.domain:type_name -> Domain + 5, // 5: GeoSiteList.entry:type_name -> MatchGeoSite + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name +} + +func init() { file_v2geo_proto_init() } +func file_v2geo_proto_init() { + if File_v2geo_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_v2geo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Domain); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CIDR); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GeoIP); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GeoIPList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GeoSite); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GeoSiteList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2geo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Domain_Attribute); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_v2geo_proto_msgTypes[6].OneofWrappers = []interface{}{ + (*Domain_Attribute_BoolValue)(nil), + (*Domain_Attribute_IntValue)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_v2geo_proto_rawDesc, + NumEnums: 1, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_v2geo_proto_goTypes, + DependencyIndexes: file_v2geo_proto_depIdxs, + EnumInfos: file_v2geo_proto_enumTypes, + MessageInfos: file_v2geo_proto_msgTypes, + }.Build() + File_v2geo_proto = out.File + file_v2geo_proto_rawDesc = nil + file_v2geo_proto_goTypes = nil + file_v2geo_proto_depIdxs = nil +} diff --git a/ruleset/builtins/geo/v2geo/v2geo.proto b/ruleset/builtins/geo/v2geo/v2geo.proto new file mode 100644 index 0000000..48ff6f0 --- /dev/null +++ b/ruleset/builtins/geo/v2geo/v2geo.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; + +option go_package = "./v2geo"; + +// This file is copied from +// https://github.com/v2fly/v2ray-core/blob/master/app/router/routercommon/common.proto +// with some modifications. + +// Domain for routing decision. +message Domain { + // Type of domain value. + enum Type { + // The value is used as is. + Plain = 0; + // The value is used as a regular expression. + Regex = 1; + // The value is a root domain. + RootDomain = 2; + // The value is a domain. + Full = 3; + } + + // Domain matching type. + Type type = 1; + + // Domain value. + string value = 2; + + message Attribute { + string key = 1; + + oneof typed_value { + bool bool_value = 2; + int64 int_value = 3; + } + } + + // Attributes of this domain. May be used for filtering. + repeated Attribute attribute = 3; +} + +// IP for routing decision, in CIDR form. +message CIDR { + // IP address, should be either 4 or 16 bytes. + bytes ip = 1; + + // Number of leading ones in the network mask. + uint32 prefix = 2; +} + +message GeoIP { + string country_code = 1; + repeated CIDR cidr = 2; + bool inverse_match = 3; + + // resource_hash instruct simplified config converter to load domain from geo file. + bytes resource_hash = 4; + string code = 5; +} + +message GeoIPList { + repeated GeoIP entry = 1; +} + +message GeoSite { + string country_code = 1; + repeated Domain domain = 2; + + // resource_hash instruct simplified config converter to load domain from geo file. + bytes resource_hash = 3; + string code = 4; +} + +message GeoSiteList { + repeated GeoSite entry = 1; +} diff --git a/ruleset/expr.go b/ruleset/expr.go index 45b7e05..5bcfb4d 100644 --- a/ruleset/expr.go +++ b/ruleset/expr.go @@ -14,6 +14,7 @@ import ( "github.com/apernet/OpenGFW/analyzer" "github.com/apernet/OpenGFW/modifier" + "github.com/apernet/OpenGFW/ruleset/builtins/geo" ) // ExprRule is the external representation of an expression rule. @@ -51,8 +52,9 @@ type compiledExprRule struct { var _ Ruleset = (*exprRuleset)(nil) type exprRuleset struct { - Rules []compiledExprRule - Ans []analyzer.Analyzer + Rules []compiledExprRule + Ans []analyzer.Analyzer + GeoMatcher *geo.GeoMatcher } func (r *exprRuleset) Analyzers(info StreamInfo) []analyzer.Analyzer { @@ -83,11 +85,15 @@ func (r *exprRuleset) Match(info StreamInfo) (MatchResult, error) { // CompileExprRules compiles a list of expression rules into a ruleset. // It returns an error if any of the rules are invalid, or if any of the analyzers // used by the rules are unknown (not provided in the analyzer list). -func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier.Modifier) (Ruleset, error) { +func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier.Modifier, config *BuiltinConfig) (Ruleset, error) { var compiledRules []compiledExprRule fullAnMap := analyzersToMap(ans) fullModMap := modifiersToMap(mods) depAnMap := make(map[string]analyzer.Analyzer) + geoMatcher, err := geo.NewGeoMatcher(config.GeoSiteFilename, config.GeoIpFilename) + if err != nil { + return nil, err + } // Compile all rules and build a map of analyzers that are used by the rules. for _, rule := range rules { action, ok := actionStringToAction(rule.Action) @@ -95,12 +101,28 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier return nil, fmt.Errorf("rule %q has invalid action %q", rule.Name, rule.Action) } visitor := &depVisitor{Analyzers: make(map[string]struct{})} + geoip := expr.Function( + "geoip", + func(params ...any) (any, error) { + return geoMatcher.MatchGeoIp(params[0].(string), params[1].(string)), nil + }, + new(func(string, string) bool), + ) + geosite := expr.Function( + "geosite", + func(params ...any) (any, error) { + return geoMatcher.MatchGeoSite(params[0].(string), params[1].(string)), nil + }, + new(func(string, string) bool), + ) program, err := expr.Compile(rule.Expr, func(c *conf.Config) { c.Strict = false c.Expect = reflect.Bool c.Visitors = append(c.Visitors, visitor) }, + geoip, + geosite, ) if err != nil { return nil, fmt.Errorf("rule %q has invalid expression: %w", rule.Name, err) @@ -112,6 +134,16 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier } depAnMap[name] = a } + if visitor.UseGeoSite { + if err := geoMatcher.LoadGeoSite(); err != nil { + return nil, fmt.Errorf("rule %q failed to load geosite: %w", rule.Name, err) + } + } + if visitor.UseGeoIp { + if err := geoMatcher.LoadGeoIP(); err != nil { + return nil, fmt.Errorf("rule %q failed to load geoip: %w", rule.Name, err) + } + } cr := compiledExprRule{ Name: rule.Name, Action: action, @@ -137,8 +169,9 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier depAns = append(depAns, a) } return &exprRuleset{ - Rules: compiledRules, - Ans: depAns, + Rules: compiledRules, + Ans: depAns, + GeoMatcher: geoMatcher, }, nil } @@ -210,10 +243,20 @@ func modifiersToMap(mods []modifier.Modifier) map[string]modifier.Modifier { type depVisitor struct { Analyzers map[string]struct{} + + UseGeoSite bool + UseGeoIp bool } func (v *depVisitor) Visit(node *ast.Node) { if idNode, ok := (*node).(*ast.IdentifierNode); ok { - v.Analyzers[idNode.Value] = struct{}{} + switch idNode.Value { + case "geosite": + v.UseGeoSite = true + case "geoip": + v.UseGeoIp = true + default: + v.Analyzers[idNode.Value] = struct{}{} + } } } diff --git a/ruleset/interface.go b/ruleset/interface.go index 30bece7..7469d78 100644 --- a/ruleset/interface.go +++ b/ruleset/interface.go @@ -92,3 +92,8 @@ type Ruleset interface { // It must be safe for concurrent use by multiple workers. Match(StreamInfo) (MatchResult, error) } + +type BuiltinConfig struct { + GeoSiteFilename string + GeoIpFilename string +}