commit d82867409f5e4849cf926fafe39b983550876f8e Author: Maurizio Porrato Date: Sat Sep 17 09:51:33 2022 +0100 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84c3191 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*[~%] +*.swp diff --git a/README.md b/README.md new file mode 100644 index 0000000..d34c623 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# squid-rewriter + +This is a rewrite engine for squid. It is meant to be used with the `url_rewrite_program` directive in your `squid.conf` file. + +The rewrite rules are read from a yaml file. The file is expected to look like something like this: + +```yaml +--- +rewrites: + - name: static + urls: + - http://proxy/ + destination: http://127.0.0.1:9999/ + - name: fromfile + filename: myrewrites.list + destination: http://example.com/ + - name: alpine + distro: alpine + destination: https://dl-cdn.alpinelinux.org/alpine/ + - name: debian + distro: debian + destination: http://deb.debian.org/debian/ + - name: arch + distro: arch + destination: https://geo.mirror.pkgbuild.com/ + - name: fedora + distro: fedora + destination: https://fedora.mirrorservice.org/fedora/linux/ + - name: fedora + distro: fedora:epel + destination: https://fedora.mirrorservice.org/epel/ + - name: ubuntu + distro: ubuntu + destination: http://archive.ubuntu.com/ubuntu/ + - name: mint + distro: mint + destination: https://mirrors.layeronline.com/linuxmint/ + - name: slackware + distro: slackware + destination: https://www.mirrorservice.org/sites/ftp.slackware.com/pub/slackware/ +``` + +Each rewrite entry must contain a descriptive name (`name`) a destination url (`destination`) and one or more of the following: + +- a list of static prefixes (`urls`) +- the name of a text file containing one prefix per line (`filename`) +- the name of a distribution repo (`distro`) in the format `distroname:reponame`; if reponame is omitted, it defaults to `main`; the name of the repos depend on the distro; at the moment only the `fedora` distro supports repos other than `main` (`epel`, `centos`) + +On startup, squid-rewriter will look for rewrite rules in the following files: + +- `rewrites.yaml` in the current directory +- `rewrites.yml` in the current directory +- `/etc/squid/rewrites.yaml` +- `/etc/squid/rewrites.yml` + +Only the first existent file from the list above will be loaded. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..c32dd18 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to \ No newline at end of file diff --git a/config/loader.go b/config/loader.go new file mode 100644 index 0000000..e3d45b7 --- /dev/null +++ b/config/loader.go @@ -0,0 +1,31 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v2" +) + +type Rewrite struct { + Name string `yaml:"name"` + Destination string `yaml:"destination"` + Distro string `yaml:"distro,omitempty"` + Urls []string `yaml:"urls,omitempty"` + Filename string `yaml:"filename,omitempty"` +} + +type Config struct { + Rewrites []Rewrite `yaml:"rewrites"` +} + +func Load(filename string) (Config, error) { + cfg := Config{} + f, err := os.Open(filename) + if err != nil { + return cfg, err + } + defer f.Close() + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&cfg) + return cfg, err +} diff --git a/distro/alpine.go b/distro/alpine.go new file mode 100644 index 0000000..7c8ca8a --- /dev/null +++ b/distro/alpine.go @@ -0,0 +1,39 @@ +package distro + +import ( + "strings" + "sync" + + "github.com/gocolly/colly" +) + +type Alpine struct{} + +func (d Alpine) GetName() string { + return "alpine" +} + +func (d Alpine) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Alpine) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + c := colly.NewCollector( + colly.AllowedDomains("mirrors.alpinelinux.org"), + ) + + c.OnHTML("div.mirrors table a", func(h *colly.HTMLElement) { + url := h.Attr("href") + if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") { + if !strings.HasSuffix(url, "/") { + url += "/" + } + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url} + } + }) + + c.Visit("https://mirrors.alpinelinux.org/") + c.Wait() +} diff --git a/distro/arch.go b/distro/arch.go new file mode 100644 index 0000000..e5766e2 --- /dev/null +++ b/distro/arch.go @@ -0,0 +1,43 @@ +package distro + +import ( + "bufio" + "net/http" + "regexp" + "strings" + "sync" +) + +type Arch struct{} + +func (d Arch) GetName() string { + return "arch" +} + +func (d Arch) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Arch) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + // re := regexp.MustCompile(`^#*\s*Server\s*=\s*(\S+)\$repo/os/\$arch`) + re := regexp.MustCompile(`^#*\s*Server\s*=\s*(\S+?)[^/]+/os/[^/]+`) + + resp, err := http.Get("https://archlinux.org/mirrorlist/all/") + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + m := re.FindSubmatch([]byte(line)) + if len(m) > 1 { + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, string(m[1])} + } + } + } +} diff --git a/distro/debian.go b/distro/debian.go new file mode 100644 index 0000000..4206b5f --- /dev/null +++ b/distro/debian.go @@ -0,0 +1,41 @@ +package distro + +import ( + "strings" + "sync" + + "github.com/gocolly/colly" +) + +type Debian struct{} + +func (d Debian) GetName() string { + return "debian" +} + +func (d Debian) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Debian) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + c := colly.NewCollector( + colly.AllowedDomains("www.debian.org"), + ) + + c.OnHTML("h2#complete-list+table", func(h *colly.HTMLElement) { + h.ForEach("a", func(i int, h *colly.HTMLElement) { + url := h.Attr("href") + if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") { + if !strings.HasSuffix(url, "/") { + url += "/" + } + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url} + } + }) + }) + + c.Visit("https://www.debian.org/mirror/list") + c.Wait() +} diff --git a/distro/distro.go b/distro/distro.go new file mode 100644 index 0000000..a5c1e5e --- /dev/null +++ b/distro/distro.go @@ -0,0 +1,57 @@ +package distro + +import ( + "sync" +) + +const DefaultRepo = "main" + +type RepoMirror struct { + Distro string + Repo string + Url string +} + +type Distro interface { + GetName() string + GetRepos() []string + FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) +} + +var allDistros []Distro = []Distro{ + Alpine{}, + Arch{}, + Debian{}, + Fedora{}, + Mint{}, + Slackware{}, + Ubuntu{}, +} + +var distrosByName map[string]Distro + +func init() { + distrosByName = make(map[string]Distro) + for _, d := range allDistros { + distrosByName[d.GetName()] = d + } +} + +func GetDistro(name string) Distro { + r, ok := distrosByName[name] + if !ok { + return nil + } + + return r +} + +func HasRepo(d Distro, name string) bool { + for _, repo := range d.GetRepos() { + if repo == name { + return true + } + } + + return false +} diff --git a/distro/fedora.go b/distro/fedora.go new file mode 100644 index 0000000..420e30a --- /dev/null +++ b/distro/fedora.go @@ -0,0 +1,65 @@ +package distro + +import ( + "bufio" + "fmt" + "net/http" + "regexp" + "strings" + "sync" +) + +var repoPathByName map[string]string = map[string]string{ + "main": "fedora/", + "epel": "epel/", + "centos": "centos/", +} + +type Fedora struct{} + +func (d Fedora) GetName() string { + return "fedora" +} + +func (d Fedora) GetRepos() []string { + r := []string{} + for k := range repoPathByName { + r = append(r, k) + } + return r +} + +func (d Fedora) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + re := regexp.MustCompile(`^\S*((https?|ftp)://.*)\S*$`) + + for _, repoName := range repos { + repoPath, ok := repoPathByName[repoName] + if !ok { + continue + } + url := fmt.Sprintf("http://mirrors.fedoraproject.org/mirrorlist?path=%s", repoPath) + done.Add(1) + go func(u string, rn string) { + defer done.Done() + + resp, err := http.Get(u) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + m := re.FindSubmatch([]byte(line)) + if len(m) > 1 { + repoMirrors <- RepoMirror{d.GetName(), rn, string(m[1])} + } + } + } + }(url, repoName) + } +} diff --git a/distro/mint.go b/distro/mint.go new file mode 100644 index 0000000..b88f780 --- /dev/null +++ b/distro/mint.go @@ -0,0 +1,39 @@ +package distro + +import ( + "strings" + "sync" + + "github.com/gocolly/colly" +) + +type Mint struct{} + +func (d Mint) GetName() string { + return "mint" +} + +func (d Mint) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Mint) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + c := colly.NewCollector( + colly.AllowedDomains("linuxmint.com"), + ) + + c.OnHTML("div.container table.table tr td:nth-child(3)", func(h *colly.HTMLElement) { + url := h.Text + if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") { + if !strings.HasSuffix(url, "/") { + url += "/" + } + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url} + } + }) + + c.Visit("https://linuxmint.com/mirrors.php") + c.Wait() +} diff --git a/distro/slackware.go b/distro/slackware.go new file mode 100644 index 0000000..50f71f0 --- /dev/null +++ b/distro/slackware.go @@ -0,0 +1,39 @@ +package distro + +import ( + "strings" + "sync" + + "github.com/gocolly/colly" +) + +type Slackware struct{} + +func (d Slackware) GetName() string { + return "slackware" +} + +func (d Slackware) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Slackware) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + c := colly.NewCollector( + colly.AllowedDomains("mirrors.slackware.com"), + ) + + c.OnHTML("body.list a", func(h *colly.HTMLElement) { + url := h.Attr("href") + if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") { + if !strings.HasSuffix(url, "/") { + url += "/" + } + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url} + } + }) + + c.Visit("https://mirrors.slackware.com/mirrorlist/") + c.Wait() +} diff --git a/distro/ubuntu.go b/distro/ubuntu.go new file mode 100644 index 0000000..500297a --- /dev/null +++ b/distro/ubuntu.go @@ -0,0 +1,69 @@ +package distro + +import ( + "bufio" + "fmt" + "net/http" + "regexp" + "strings" + "sync" +) + +const ( + ubuntuMirrorsBaseUrl = "http://mirrors.ubuntu.com" +) + +type Ubuntu struct{} + +func (d Ubuntu) GetName() string { + return "ubuntu" +} + +func (d Ubuntu) GetRepos() []string { + return []string{DefaultRepo} +} + +func (d Ubuntu) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) { + defer done.Done() + + countryMirrorsRe := regexp.MustCompile(`>([A-Z]{2}\.txt)<`) + re := regexp.MustCompile(`^\S*((https?|ftp)://.*)\S*$`) + + resp, err := http.Get(ubuntuMirrorsBaseUrl) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + m := countryMirrorsRe.FindSubmatch([]byte(line)) + if len(m) > 1 { + url := fmt.Sprintf("%s/%s", ubuntuMirrorsBaseUrl, string(m[1])) + done.Add(1) + go func(u string) { + defer done.Done() + + resp, err := http.Get(u) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + m := re.FindSubmatch([]byte(line)) + if len(m) > 1 { + repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, string(m[1])} + } + } + } + }(url) + } + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..242b5f4 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module git.worn.eu/guru/squid-rewriter + +go 1.18 + +require ( + github.com/gocolly/colly v1.2.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.2.5 // indirect + github.com/antchfx/xmlquery v1.3.12 // indirect + github.com/antchfx/xpath v1.2.1 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.3.1 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/labstack/gommon v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mkideal/cli v0.2.7 // indirect + github.com/mkideal/expr v0.1.0 // indirect + github.com/mkideal/pkg v0.1.3 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/suntong/cascadia v1.2.6 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect + golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect + golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/appengine v1.6.7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..adbddbb --- /dev/null +++ b/go.sum @@ -0,0 +1,108 @@ +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= +github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw= +github.com/antchfx/xmlquery v1.3.12 h1:6TMGpdjpO/P8VhjnaYPXuqT3qyJ/VsqoyNTmJzNBTQ4= +github.com/antchfx/xmlquery v1.3.12/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ= +github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= +github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o= +github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mkideal/cli v0.2.7 h1:mB/XrMzuddmTJ8f7KY1c+KzfYoM149tYGAnzmqRdvOU= +github.com/mkideal/cli v0.2.7/go.mod h1:efaTeFI4jdPqzAe0bv3myLB2NW5yzMBLvWB70a6feco= +github.com/mkideal/expr v0.1.0 h1:fzborV9TeSUmLm0aEQWTWcexDURFFo4v5gHSc818Kl8= +github.com/mkideal/expr v0.1.0/go.mod h1:vL1DsSb87ZtU6IEjOtUfxw98z0FQbzS8xlGtnPkKdzg= +github.com/mkideal/pkg v0.1.3 h1:4XlD59fshHEiO8z7jftNHYrK7qjp5+2xK7VDnvZw0Qo= +github.com/mkideal/pkg v0.1.3/go.mod h1:u/enAxPeRcYSsxtu1NUifWSeOTU/31VsCaOPg54SMJ4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/suntong/cascadia v1.2.6 h1:qfPIQ/stZwar4I+o1ssX1W+V5mAkAsj/m5IKLWIsVz8= +github.com/suntong/cascadia v1.2.6/go.mod h1:8n7lHQtXvG8VHDo59tgmeg9nrgjvIbmjeEQprmjAq4M= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho= +golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..1656a62 --- /dev/null +++ b/main.go @@ -0,0 +1,197 @@ +package main + +import ( + "bufio" + "flag" + "log" + "os" + "strings" + "sync" + "time" + + "git.worn.eu/guru/squid-rewriter/config" + "git.worn.eu/guru/squid-rewriter/distro" + "git.worn.eu/guru/squid-rewriter/squid" + "git.worn.eu/guru/squid-rewriter/trie" +) + +var Flags struct { + ConfigFile string + Verbose bool +} + +func RewritesFromConfig(cfgfile string) (*trie.Trie, error) { + t := trie.NewTrie() + cfg, err := config.Load(cfgfile) + if err != nil { + return &t, err + } + + repoRewrites := make(map[string]map[string]string, 100) + + for _, r := range cfg.Rewrites { + if r.Destination == "" { + continue + } + if r.Distro != "" { + distroName := r.Distro + repoName := "main" + if strings.Contains(distroName, ":") { + f := strings.SplitN(distroName, ":", 2) + distroName = f[0] + repoName = f[1] + } + if _, ok := repoRewrites[distroName]; !ok { + repoRewrites[distroName] = make(map[string]string) + } + repoRewrites[distroName][repoName] = r.Destination + } + } + + if len(repoRewrites) > 0 { + repoMirrors := make(chan distro.RepoMirror) + wg := sync.WaitGroup{} + + go func() { + for r := range repoMirrors { + d, ok := repoRewrites[r.Distro] + if !ok { + continue + } + destination, ok := d[r.Repo] + if !ok { + continue + } + t.Put(r.Url, destination) + } + }() + + for distroName, repos := range repoRewrites { + repoList := []string{} + for r := range repos { + repoList = append(repoList, r) + } + d := distro.GetDistro(distroName) + if d == nil { + continue + } + log.Printf("Fetching %s mirrors %v", distroName, repoList) + wg.Add(1) + go d.FetchMirrors(repoList, repoMirrors, &wg) + } + + wg.Wait() + close(repoMirrors) + if len(repoMirrors) > 0 { + log.Println("Waiting for all repo rewrite rules to be processed...") + for len(repoMirrors) > 0 { + time.Sleep(5 * time.Millisecond) + } + } + } + + for _, r := range cfg.Rewrites { + if r.Destination == "" { + continue + } + if len(r.Urls) > 0 { + if Flags.Verbose { + log.Printf("Loading static rewites for %s", r.Name) + } + for _, url := range r.Urls { + t.Put(url, r.Destination) + } + } + if r.Filename != "" { + if Flags.Verbose { + log.Printf("Loading rewrites from %s for %s", r.Filename, r.Name) + } + f, err := os.Open(r.Filename) + if err == nil { + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" && !strings.HasPrefix(line, "#") { + if line != r.Destination { + t.Put(line, r.Destination) + } + } + } + } else { + log.Printf("Can't open file %s", r.Filename) + } + } + } + + return &t, nil +} + +func Rewrite(t *trie.Trie) { + log.Println("Listening for requests") + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + req := squid.ParseRequest(scanner.Text()) + if req == nil { + continue + } + resp := req.MakeResponse() + if req.Url != "" { + resp.Result = "OK" + prefix, dest := t.GetLongestPrefix(req.Url) + if sdest, ok := dest.(string); ok { + newUrl := sdest + req.Url[len(prefix):] + resp.RewriteTo(newUrl) + if Flags.Verbose { + log.Printf("Rewriting %s to %s", req.Url, newUrl) + } + } + } else { + resp.Result = "ERR" + } + os.Stdout.WriteString(resp.Format() + "\n") + } + log.Println("End of input stream. Exiting.") +} + +func init() { + flag.StringVar(&Flags.ConfigFile, "c", "", "Path to rewrite config file") + flag.BoolVar(&Flags.Verbose, "v", false, "Verbose logging") +} + +var RewriteFileNames []string = []string{"rewrites.yaml", "rewrites.yml", "/etc/squid/rewrites.yaml", "/etc/squid/rewrites.yml"} + +func LoadRewrites() *trie.Trie { + var t *trie.Trie + var err error + + start := time.Now() + if Flags.ConfigFile != "" { + t, err = RewritesFromConfig(Flags.ConfigFile) + if err != nil { + log.Fatalf("Can't load the specified rewrite file: %s", Flags.ConfigFile) + } + } else { + for _, filename := range RewriteFileNames { + if Flags.Verbose { + log.Printf("Trying to load rewrites from %s", filename) + } + t, err = RewritesFromConfig(filename) + if err == nil { + break + } + } + if err != nil { + log.Fatalf("Can't load any of the predefined rewrite files: %v", RewriteFileNames) + } + } + elapsed := time.Since(start) + log.Printf("Loaded %d unique rewrites in %s", t.Count(), elapsed) + + return t +} + +func main() { + flag.Parse() + Rewrite(LoadRewrites()) +} diff --git a/rewrites.yaml.sample b/rewrites.yaml.sample new file mode 100644 index 0000000..55246d8 --- /dev/null +++ b/rewrites.yaml.sample @@ -0,0 +1,34 @@ +--- +# This is an example rewrite rules file +rewrites: + - name: static + urls: + - http://proxy/ + destination: http://127.0.0.1:9999/ + - name: fromfile + filename: myrewrites.list + destination: http://example.com/ + - name: alpine + distro: alpine + destination: https://dl-cdn.alpinelinux.org/alpine/ + - name: debian + distro: debian + destination: http://deb.debian.org/debian/ + - name: arch + distro: arch + destination: https://geo.mirror.pkgbuild.com/ + - name: fedora + distro: fedora + destination: https://fedora.mirrorservice.org/fedora/linux/ + - name: fedora + distro: fedora:epel + destination: https://fedora.mirrorservice.org/epel/ + - name: ubuntu + distro: ubuntu + destination: http://archive.ubuntu.com/ubuntu/ + - name: mint + distro: mint + destination: https://mirrors.layeronline.com/linuxmint/ + - name: slackware + distro: slackware + destination: https://www.mirrorservice.org/sites/ftp.slackware.com/pub/slackware/ \ No newline at end of file diff --git a/squid/request.go b/squid/request.go new file mode 100644 index 0000000..c700786 --- /dev/null +++ b/squid/request.go @@ -0,0 +1,46 @@ +package squid + +import ( + "strconv" + "strings" +) + +type Request struct { + Id int + Url string + Args map[string]string +} + +func ParseRequest(line string) *Request { + fields := strings.Fields(line) + + if len(fields) < 1 { + return nil + } + + r := Request{} + r.Id = -1 + + for i, field := range fields { + if i == 0 { + if val, err := strconv.Atoi(field); err == nil { + r.Id = val + continue + } + } + if r.Url == "" { + r.Url = field + continue + } + if strings.Contains(field, "=") { + kv := strings.SplitN(field, "=", 2) + r.Args[kv[0]] = kv[1] + } + } + + return &r +} + +func (r *Request) MakeResponse() *Response { + return &Response{Id: r.Id} +} diff --git a/squid/response.go b/squid/response.go new file mode 100644 index 0000000..32e1d80 --- /dev/null +++ b/squid/response.go @@ -0,0 +1,32 @@ +package squid + +import "strconv" + +type Response struct { + Id int + Result string + Args map[string]string +} + +func (r *Response) Format() string { + line := "" + if r.Id >= 0 { + line += strconv.Itoa(r.Id) + " " + } + line += r.Result + for k, v := range r.Args { + line += " " + k + "=" + v + } + return line +} + +func (r *Response) SetArg(arg string, value string) { + if r.Args == nil { + r.Args = make(map[string]string) + } + r.Args[arg] = value +} + +func (r *Response) RewriteTo(dest string) { + r.SetArg("rewrite-url", dest) +} diff --git a/trie/trie.go b/trie/trie.go new file mode 100644 index 0000000..21356f6 --- /dev/null +++ b/trie/trie.go @@ -0,0 +1,147 @@ +package trie + +func commonPrefixLength(s1 string, s2 string) int { + var i, l int + + l1 := len(s1) + l2 := len(s2) + if l1 < l2 { + l = l1 + } else { + l = l2 + } + + for i = 0; i < l && s1[i] == s2[i]; i++ { + } + + return i +} + +type Trie struct { + value interface{} + children map[string]*Trie +} + +func NewTrie() Trie { + return Trie{children: make(map[string]*Trie)} +} + +func (t *Trie) Count() int { + r := 0 + if t.value != nil { + r++ + } + + for _, subTrie := range t.children { + r += subTrie.Count() + } + + return r +} + +func (t *Trie) Put(key string, value interface{}) { + currentKey := key + currentTrie := t + + for currentKey != "" { + found := false + for k, v := range currentTrie.children { + keyLen := len(k) + prefixLen := commonPrefixLength(currentKey, k) + if prefixLen == keyLen { + currentKey = currentKey[keyLen:] + currentTrie = v + found = true + break + } + if prefixLen > 0 { + commonPrefix := k[:prefixLen] + oldSuffix := k[prefixLen:] + intermediate := NewTrie() + intermediate.children[oldSuffix] = v + currentTrie.children[commonPrefix] = &intermediate + delete(currentTrie.children, k) + currentTrie = &intermediate + currentKey = currentKey[prefixLen:] + break + } + } + if !found { + break + } + } + + if currentKey == "" { + currentTrie.value = value + } else { + n := NewTrie() + n.value = value + currentTrie.children[currentKey] = &n + } +} + +func (t *Trie) Get(key string) (interface{}, bool) { + currentKey := key + currentTrie := t + + for currentKey != "" { + found := false + for k, v := range currentTrie.children { + keyLen := len(k) + prefixLen := commonPrefixLength(currentKey, k) + if prefixLen == keyLen { + currentKey = currentKey[keyLen:] + currentTrie = v + found = true + break + } + if prefixLen > 0 { + return nil, false + } + } + if !found { + break + } + } + + if currentKey == "" { + return currentTrie.value, currentTrie.value != nil + } + + return nil, false +} + +func (t *Trie) GetLongestPrefix(key string) (string, interface{}) { + currentKey := key + currentTrie := t + matchedPrefix := "" + longestPrefix := "" + var matchedValue interface{} + + for currentKey != "" { + found := false + for k, v := range currentTrie.children { + keyLen := len(k) + prefixLen := commonPrefixLength(currentKey, k) + if prefixLen == keyLen { + currentKey = currentKey[keyLen:] + currentTrie = v + matchedPrefix += k + if v.value != nil { + longestPrefix = matchedPrefix + matchedValue = v.value + } + found = true + break + } + if prefixLen > 0 { + return longestPrefix, matchedValue + } + } + if !found { + break + } + } + + return longestPrefix, matchedValue +} diff --git a/trie/trie_test.go b/trie/trie_test.go new file mode 100644 index 0000000..a148164 --- /dev/null +++ b/trie/trie_test.go @@ -0,0 +1,118 @@ +package trie + +import "testing" + +func TestNewTrie(t *testing.T) { + tt := NewTrie() + + if len(tt.children) != 0 { + t.Log("New trie should be empty") + t.Fail() + } + + if tt.value != nil { + t.Log("New trie should not have a value") + t.Fail() + } +} + +func TestPutAndCount(t *testing.T) { + tt := NewTrie() + + if tt.Count() != 0 { + t.Log("New trie should be empty") + t.Fail() + } + + tt.Put("hello", 1) + if tt.Count() != 1 { + t.Log("One element") + t.Fail() + } + + tt.Put("hello", 2) + if tt.Count() != 1 { + t.Log("Duplicate element") + t.Fail() + } + + tt.Put("here", 3) + if tt.Count() != 2 { + t.Log("Two elements") + t.Fail() + } + + tt.Put("there", 3) + if tt.Count() != 3 { + t.Log("Three elements") + t.Fail() + } +} + +func TestPutAndGet(t *testing.T) { + tt := NewTrie() + + if _, ok := tt.Get("missing"); ok { + t.Log("New trie should be empty") + t.Fail() + } + + if _, ok := tt.Get(""); ok { + t.Log("New trie should be empty") + t.Fail() + } + + tt.Put("hello", 1) + if r, ok := tt.Get("hello"); !ok || r != 1 { + t.Log("hello") + t.Fail() + } + + tt.Put("help", 2) + if r, ok := tt.Get("help"); !ok || r != 2 { + t.Log("help") + t.Fail() + } + + tt.Put("hi", 3) + if r, ok := tt.Get("hi"); !ok || r != 3 { + t.Log("hi") + t.Fail() + } +} + +func TestPutAndGetLongestPrefix(t *testing.T) { + tt := NewTrie() + + if _, value := tt.GetLongestPrefix(""); value != nil { + t.Log("New trie should be empty") + t.Fail() + } + + tt.Put("integer", 1) + tt.Put("interval", 2) + tt.Put("in", 3) + tt.Put("string", 4) + tt.Put("struct", 5) + tt.Put("stop", 6) + + if prefix, value := tt.GetLongestPrefix("interrupt"); prefix != "in" || value != 3 { + t.Log("Searching \"interrupt\"", prefix, value) + t.Fail() + } + + if prefix, value := tt.GetLongestPrefix("str"); prefix != "" || value != nil { + t.Log("Searching \"str\"", prefix, value) + t.Fail() + } + + if prefix, value := tt.GetLongestPrefix("struct"); prefix != "struct" || value != 5 { + t.Log("Searching \"struct\"", prefix, value) + t.Fail() + } + + if prefix, value := tt.GetLongestPrefix("structure"); prefix != "struct" || value != 5 { + t.Log("Searching \"structure\"", prefix, value) + t.Fail() + } +}