guru
/
squid-rewriter
1
0
Fork 0
Code Issues Releases Activity

First commit

This commit is contained in:
Maurizio Porrato 2022-09-17 09:51:33 +01:00
commit d82867409f
20 changed files with 1222 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*[~%]
*.swp

56
README.md Normal file
View File

@ -0,0 +1,56 @@
# squid-rewriter
This is a rewrite engine for squid. It is meant to be used with the `url_rewrite_program` directive in your `squid.conf` file.
The rewrite rules are read from a yaml file. The file is expected to look like something like this:
```yaml
---
rewrites:
- name: static
urls:
- http://proxy/
destination: http://127.0.0.1:9999/
- name: fromfile
filename: myrewrites.list
destination: http://example.com/
- name: alpine
distro: alpine
destination: https://dl-cdn.alpinelinux.org/alpine/
- name: debian
distro: debian
destination: http://deb.debian.org/debian/
- name: arch
distro: arch
destination: https://geo.mirror.pkgbuild.com/
- name: fedora
distro: fedora
destination: https://fedora.mirrorservice.org/fedora/linux/
- name: fedora
distro: fedora:epel
destination: https://fedora.mirrorservice.org/epel/
- name: ubuntu
distro: ubuntu
destination: http://archive.ubuntu.com/ubuntu/
- name: mint
distro: mint
destination: https://mirrors.layeronline.com/linuxmint/
- name: slackware
distro: slackware
destination: https://www.mirrorservice.org/sites/ftp.slackware.com/pub/slackware/
```
Each rewrite entry must contain a descriptive name (`name`) a destination url (`destination`) and one or more of the following:
- a list of static prefixes (`urls`)
- the name of a text file containing one prefix per line (`filename`)
- the name of a distribution repo (`distro`) in the format `distroname:reponame`; if reponame is omitted, it defaults to `main`; the name of the repos depend on the distro; at the moment only the `fedora` distro supports repos other than `main` (`epel`, `centos`)
On startup, squid-rewriter will look for rewrite rules in the following files:
- `rewrites.yaml` in the current directory
- `rewrites.yml` in the current directory
- `/etc/squid/rewrites.yaml`
- `/etc/squid/rewrites.yml`
Only the first existent file from the list above will be loaded.

24
UNLICENSE Normal file
View File

@ -0,0 +1,24 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <https://unlicense.org/>

31
config/loader.go Normal file
View File

@ -0,0 +1,31 @@
package config
import (
"os"
"gopkg.in/yaml.v2"
)
type Rewrite struct {
Name string `yaml:"name"`
Destination string `yaml:"destination"`
Distro string `yaml:"distro,omitempty"`
Urls []string `yaml:"urls,omitempty"`
Filename string `yaml:"filename,omitempty"`
}
type Config struct {
Rewrites []Rewrite `yaml:"rewrites"`
}
func Load(filename string) (Config, error) {
cfg := Config{}
f, err := os.Open(filename)
if err != nil {
return cfg, err
}
defer f.Close()
decoder := yaml.NewDecoder(f)
err = decoder.Decode(&cfg)
return cfg, err
}

39
distro/alpine.go Normal file
View File

@ -0,0 +1,39 @@
package distro
import (
"strings"
"sync"
"github.com/gocolly/colly"
)
type Alpine struct{}
func (d Alpine) GetName() string {
return "alpine"
}
func (d Alpine) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Alpine) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
c := colly.NewCollector(
colly.AllowedDomains("mirrors.alpinelinux.org"),
)
c.OnHTML("div.mirrors table a", func(h *colly.HTMLElement) {
url := h.Attr("href")
if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") {
if !strings.HasSuffix(url, "/") {
url += "/"
}
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url}
}
})
c.Visit("https://mirrors.alpinelinux.org/")
c.Wait()
}

43
distro/arch.go Normal file
View File

@ -0,0 +1,43 @@
package distro
import (
"bufio"
"net/http"
"regexp"
"strings"
"sync"
)
type Arch struct{}
func (d Arch) GetName() string {
return "arch"
}
func (d Arch) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Arch) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
// re := regexp.MustCompile(`^#*\s*Server\s*=\s*(\S+)\$repo/os/\$arch`)
re := regexp.MustCompile(`^#*\s*Server\s*=\s*(\S+?)[^/]+/os/[^/]+`)
resp, err := http.Get("https://archlinux.org/mirrorlist/all/")
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
m := re.FindSubmatch([]byte(line))
if len(m) > 1 {
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, string(m[1])}
}
}
}
}

41
distro/debian.go Normal file
View File

@ -0,0 +1,41 @@
package distro
import (
"strings"
"sync"
"github.com/gocolly/colly"
)
type Debian struct{}
func (d Debian) GetName() string {
return "debian"
}
func (d Debian) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Debian) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
c := colly.NewCollector(
colly.AllowedDomains("www.debian.org"),
)
c.OnHTML("h2#complete-list+table", func(h *colly.HTMLElement) {
h.ForEach("a", func(i int, h *colly.HTMLElement) {
url := h.Attr("href")
if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") {
if !strings.HasSuffix(url, "/") {
url += "/"
}
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url}
}
})
})
c.Visit("https://www.debian.org/mirror/list")
c.Wait()
}

57
distro/distro.go Normal file
View File

@ -0,0 +1,57 @@
package distro
import (
"sync"
)
const DefaultRepo = "main"
type RepoMirror struct {
Distro string
Repo string
Url string
}
type Distro interface {
GetName() string
GetRepos() []string
FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup)
}
var allDistros []Distro = []Distro{
Alpine{},
Arch{},
Debian{},
Fedora{},
Mint{},
Slackware{},
Ubuntu{},
}
var distrosByName map[string]Distro
func init() {
distrosByName = make(map[string]Distro)
for _, d := range allDistros {
distrosByName[d.GetName()] = d
}
}
func GetDistro(name string) Distro {
r, ok := distrosByName[name]
if !ok {
return nil
}
return r
}
func HasRepo(d Distro, name string) bool {
for _, repo := range d.GetRepos() {
if repo == name {
return true
}
}
return false
}

65
distro/fedora.go Normal file
View File

@ -0,0 +1,65 @@
package distro
import (
"bufio"
"fmt"
"net/http"
"regexp"
"strings"
"sync"
)
var repoPathByName map[string]string = map[string]string{
"main": "fedora/",
"epel": "epel/",
"centos": "centos/",
}
type Fedora struct{}
func (d Fedora) GetName() string {
return "fedora"
}
func (d Fedora) GetRepos() []string {
r := []string{}
for k := range repoPathByName {
r = append(r, k)
}
return r
}
func (d Fedora) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
re := regexp.MustCompile(`^\S*((https?|ftp)://.*)\S*$`)
for _, repoName := range repos {
repoPath, ok := repoPathByName[repoName]
if !ok {
continue
}
url := fmt.Sprintf("http://mirrors.fedoraproject.org/mirrorlist?path=%s", repoPath)
done.Add(1)
go func(u string, rn string) {
defer done.Done()
resp, err := http.Get(u)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
m := re.FindSubmatch([]byte(line))
if len(m) > 1 {
repoMirrors <- RepoMirror{d.GetName(), rn, string(m[1])}
}
}
}
}(url, repoName)
}
}

39
distro/mint.go Normal file
View File

@ -0,0 +1,39 @@
package distro
import (
"strings"
"sync"
"github.com/gocolly/colly"
)
type Mint struct{}
func (d Mint) GetName() string {
return "mint"
}
func (d Mint) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Mint) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
c := colly.NewCollector(
colly.AllowedDomains("linuxmint.com"),
)
c.OnHTML("div.container table.table tr td:nth-child(3)", func(h *colly.HTMLElement) {
url := h.Text
if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") {
if !strings.HasSuffix(url, "/") {
url += "/"
}
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url}
}
})
c.Visit("https://linuxmint.com/mirrors.php")
c.Wait()
}

39
distro/slackware.go Normal file
View File

@ -0,0 +1,39 @@
package distro
import (
"strings"
"sync"
"github.com/gocolly/colly"
)
type Slackware struct{}
func (d Slackware) GetName() string {
return "slackware"
}
func (d Slackware) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Slackware) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
c := colly.NewCollector(
colly.AllowedDomains("mirrors.slackware.com"),
)
c.OnHTML("body.list a", func(h *colly.HTMLElement) {
url := h.Attr("href")
if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") || strings.HasPrefix(url, "ftp:") {
if !strings.HasSuffix(url, "/") {
url += "/"
}
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, url}
}
})
c.Visit("https://mirrors.slackware.com/mirrorlist/")
c.Wait()
}

69
distro/ubuntu.go Normal file
View File

@ -0,0 +1,69 @@
package distro
import (
"bufio"
"fmt"
"net/http"
"regexp"
"strings"
"sync"
)
const (
ubuntuMirrorsBaseUrl = "http://mirrors.ubuntu.com"
)
type Ubuntu struct{}
func (d Ubuntu) GetName() string {
return "ubuntu"
}
func (d Ubuntu) GetRepos() []string {
return []string{DefaultRepo}
}
func (d Ubuntu) FetchMirrors(repos []string, repoMirrors chan<- RepoMirror, done *sync.WaitGroup) {
defer done.Done()
countryMirrorsRe := regexp.MustCompile(`>([A-Z]{2}\.txt)<`)
re := regexp.MustCompile(`^\S*((https?|ftp)://.*)\S*$`)
resp, err := http.Get(ubuntuMirrorsBaseUrl)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
m := countryMirrorsRe.FindSubmatch([]byte(line))
if len(m) > 1 {
url := fmt.Sprintf("%s/%s", ubuntuMirrorsBaseUrl, string(m[1]))
done.Add(1)
go func(u string) {
defer done.Done()
resp, err := http.Get(u)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
m := re.FindSubmatch([]byte(line))
if len(m) > 1 {
repoMirrors <- RepoMirror{d.GetName(), DefaultRepo, string(m[1])}
}
}
}
}(url)
}
}
}
}

35
go.mod Normal file
View File

@ -0,0 +1,35 @@
module git.worn.eu/guru/squid-rewriter
go 1.18
require (
github.com/gocolly/colly v1.2.0
gopkg.in/yaml.v2 v2.4.0
)
require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/antchfx/htmlquery v1.2.5 // indirect
github.com/antchfx/xmlquery v1.3.12 // indirect
github.com/antchfx/xpath v1.2.1 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/labstack/gommon v0.3.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mkideal/cli v0.2.7 // indirect
github.com/mkideal/expr v0.1.0 // indirect
github.com/mkideal/pkg v0.1.3 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/suntong/cascadia v1.2.6 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect
)

108
go.sum Normal file
View File

@ -0,0 +1,108 @@
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4=
github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw=
github.com/antchfx/xmlquery v1.3.12 h1:6TMGpdjpO/P8VhjnaYPXuqT3qyJ/VsqoyNTmJzNBTQ4=
github.com/antchfx/xmlquery v1.3.12/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ=
github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8=
github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o=
github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mkideal/cli v0.2.7 h1:mB/XrMzuddmTJ8f7KY1c+KzfYoM149tYGAnzmqRdvOU=
github.com/mkideal/cli v0.2.7/go.mod h1:efaTeFI4jdPqzAe0bv3myLB2NW5yzMBLvWB70a6feco=
github.com/mkideal/expr v0.1.0 h1:fzborV9TeSUmLm0aEQWTWcexDURFFo4v5gHSc818Kl8=
github.com/mkideal/expr v0.1.0/go.mod h1:vL1DsSb87ZtU6IEjOtUfxw98z0FQbzS8xlGtnPkKdzg=
github.com/mkideal/pkg v0.1.3 h1:4XlD59fshHEiO8z7jftNHYrK7qjp5+2xK7VDnvZw0Qo=
github.com/mkideal/pkg v0.1.3/go.mod h1:u/enAxPeRcYSsxtu1NUifWSeOTU/31VsCaOPg54SMJ4=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/suntong/cascadia v1.2.6 h1:qfPIQ/stZwar4I+o1ssX1W+V5mAkAsj/m5IKLWIsVz8=
github.com/suntong/cascadia v1.2.6/go.mod h1:8n7lHQtXvG8VHDo59tgmeg9nrgjvIbmjeEQprmjAq4M=
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

197
main.go Normal file
View File

@ -0,0 +1,197 @@
package main
import (
"bufio"
"flag"
"log"
"os"
"strings"
"sync"
"time"
"git.worn.eu/guru/squid-rewriter/config"
"git.worn.eu/guru/squid-rewriter/distro"
"git.worn.eu/guru/squid-rewriter/squid"
"git.worn.eu/guru/squid-rewriter/trie"
)
var Flags struct {
ConfigFile string
Verbose bool
}
func RewritesFromConfig(cfgfile string) (*trie.Trie, error) {
t := trie.NewTrie()
cfg, err := config.Load(cfgfile)
if err != nil {
return &t, err
}
repoRewrites := make(map[string]map[string]string, 100)
for _, r := range cfg.Rewrites {
if r.Destination == "" {
continue
}
if r.Distro != "" {
distroName := r.Distro
repoName := "main"
if strings.Contains(distroName, ":") {
f := strings.SplitN(distroName, ":", 2)
distroName = f[0]
repoName = f[1]
}
if _, ok := repoRewrites[distroName]; !ok {
repoRewrites[distroName] = make(map[string]string)
}
repoRewrites[distroName][repoName] = r.Destination
}
}
if len(repoRewrites) > 0 {
repoMirrors := make(chan distro.RepoMirror)
wg := sync.WaitGroup{}
go func() {
for r := range repoMirrors {
d, ok := repoRewrites[r.Distro]
if !ok {
continue
}
destination, ok := d[r.Repo]
if !ok {
continue
}
t.Put(r.Url, destination)
}
}()
for distroName, repos := range repoRewrites {
repoList := []string{}
for r := range repos {
repoList = append(repoList, r)
}
d := distro.GetDistro(distroName)
if d == nil {
continue
}
log.Printf("Fetching %s mirrors %v", distroName, repoList)
wg.Add(1)
go d.FetchMirrors(repoList, repoMirrors, &wg)
}
wg.Wait()
close(repoMirrors)
if len(repoMirrors) > 0 {
log.Println("Waiting for all repo rewrite rules to be processed...")
for len(repoMirrors) > 0 {
time.Sleep(5 * time.Millisecond)
}
}
}
for _, r := range cfg.Rewrites {
if r.Destination == "" {
continue
}
if len(r.Urls) > 0 {
if Flags.Verbose {
log.Printf("Loading static rewites for %s", r.Name)
}
for _, url := range r.Urls {
t.Put(url, r.Destination)
}
}
if r.Filename != "" {
if Flags.Verbose {
log.Printf("Loading rewrites from %s for %s", r.Filename, r.Name)
}
f, err := os.Open(r.Filename)
if err == nil {
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" && !strings.HasPrefix(line, "#") {
if line != r.Destination {
t.Put(line, r.Destination)
}
}
}
} else {
log.Printf("Can't open file %s", r.Filename)
}
}
}
return &t, nil
}
func Rewrite(t *trie.Trie) {
log.Println("Listening for requests")
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
req := squid.ParseRequest(scanner.Text())
if req == nil {
continue
}
resp := req.MakeResponse()
if req.Url != "" {
resp.Result = "OK"
prefix, dest := t.GetLongestPrefix(req.Url)
if sdest, ok := dest.(string); ok {
newUrl := sdest + req.Url[len(prefix):]
resp.RewriteTo(newUrl)
if Flags.Verbose {
log.Printf("Rewriting %s to %s", req.Url, newUrl)
}
}
} else {
resp.Result = "ERR"
}
os.Stdout.WriteString(resp.Format() + "\n")
}
log.Println("End of input stream. Exiting.")
}
func init() {
flag.StringVar(&Flags.ConfigFile, "c", "", "Path to rewrite config file")
flag.BoolVar(&Flags.Verbose, "v", false, "Verbose logging")
}
var RewriteFileNames []string = []string{"rewrites.yaml", "rewrites.yml", "/etc/squid/rewrites.yaml", "/etc/squid/rewrites.yml"}
func LoadRewrites() *trie.Trie {
var t *trie.Trie
var err error
start := time.Now()
if Flags.ConfigFile != "" {
t, err = RewritesFromConfig(Flags.ConfigFile)
if err != nil {
log.Fatalf("Can't load the specified rewrite file: %s", Flags.ConfigFile)
}
} else {
for _, filename := range RewriteFileNames {
if Flags.Verbose {
log.Printf("Trying to load rewrites from %s", filename)
}
t, err = RewritesFromConfig(filename)
if err == nil {
break
}
}
if err != nil {
log.Fatalf("Can't load any of the predefined rewrite files: %v", RewriteFileNames)
}
}
elapsed := time.Since(start)
log.Printf("Loaded %d unique rewrites in %s", t.Count(), elapsed)
return t
}
func main() {
flag.Parse()
Rewrite(LoadRewrites())
}

34
rewrites.yaml.sample Normal file
View File

@ -0,0 +1,34 @@
---
# This is an example rewrite rules file
rewrites:
- name: static
urls:
- http://proxy/
destination: http://127.0.0.1:9999/
- name: fromfile
filename: myrewrites.list
destination: http://example.com/
- name: alpine
distro: alpine
destination: https://dl-cdn.alpinelinux.org/alpine/
- name: debian
distro: debian
destination: http://deb.debian.org/debian/
- name: arch
distro: arch
destination: https://geo.mirror.pkgbuild.com/
- name: fedora
distro: fedora
destination: https://fedora.mirrorservice.org/fedora/linux/
- name: fedora
distro: fedora:epel
destination: https://fedora.mirrorservice.org/epel/
- name: ubuntu
distro: ubuntu
destination: http://archive.ubuntu.com/ubuntu/
- name: mint
distro: mint
destination: https://mirrors.layeronline.com/linuxmint/
- name: slackware
distro: slackware
destination: https://www.mirrorservice.org/sites/ftp.slackware.com/pub/slackware/

46
squid/request.go Normal file
View File

@ -0,0 +1,46 @@
package squid
import (
"strconv"
"strings"
)
type Request struct {
Id int
Url string
Args map[string]string
}
func ParseRequest(line string) *Request {
fields := strings.Fields(line)
if len(fields) < 1 {
return nil
}
r := Request{}
r.Id = -1
for i, field := range fields {
if i == 0 {
if val, err := strconv.Atoi(field); err == nil {
r.Id = val
continue
}
}
if r.Url == "" {
r.Url = field
continue
}
if strings.Contains(field, "=") {
kv := strings.SplitN(field, "=", 2)
r.Args[kv[0]] = kv[1]
}
}
return &r
}
func (r *Request) MakeResponse() *Response {
return &Response{Id: r.Id}
}

32
squid/response.go Normal file
View File

@ -0,0 +1,32 @@
package squid
import "strconv"
type Response struct {
Id int
Result string
Args map[string]string
}
func (r *Response) Format() string {
line := ""
if r.Id >= 0 {
line += strconv.Itoa(r.Id) + " "
}
line += r.Result
for k, v := range r.Args {
line += " " + k + "=" + v
}
return line
}
func (r *Response) SetArg(arg string, value string) {
if r.Args == nil {
r.Args = make(map[string]string)
}
r.Args[arg] = value
}
func (r *Response) RewriteTo(dest string) {
r.SetArg("rewrite-url", dest)
}

147
trie/trie.go Normal file
View File

@ -0,0 +1,147 @@
package trie
func commonPrefixLength(s1 string, s2 string) int {
var i, l int
l1 := len(s1)
l2 := len(s2)
if l1 < l2 {
l = l1
} else {
l = l2
}
for i = 0; i < l && s1[i] == s2[i]; i++ {
}
return i
}
type Trie struct {
value interface{}
children map[string]*Trie
}
func NewTrie() Trie {
return Trie{children: make(map[string]*Trie)}
}
func (t *Trie) Count() int {
r := 0
if t.value != nil {
r++
}
for _, subTrie := range t.children {
r += subTrie.Count()
}
return r
}
func (t *Trie) Put(key string, value interface{}) {
currentKey := key
currentTrie := t
for currentKey != "" {
found := false
for k, v := range currentTrie.children {
keyLen := len(k)
prefixLen := commonPrefixLength(currentKey, k)
if prefixLen == keyLen {
currentKey = currentKey[keyLen:]
currentTrie = v
found = true
break
}
if prefixLen > 0 {
commonPrefix := k[:prefixLen]
oldSuffix := k[prefixLen:]
intermediate := NewTrie()
intermediate.children[oldSuffix] = v
currentTrie.children[commonPrefix] = &intermediate
delete(currentTrie.children, k)
currentTrie = &intermediate
currentKey = currentKey[prefixLen:]
break
}
}
if !found {
break
}
}
if currentKey == "" {
currentTrie.value = value
} else {
n := NewTrie()
n.value = value
currentTrie.children[currentKey] = &n
}
}
func (t *Trie) Get(key string) (interface{}, bool) {
currentKey := key
currentTrie := t
for currentKey != "" {
found := false
for k, v := range currentTrie.children {
keyLen := len(k)
prefixLen := commonPrefixLength(currentKey, k)
if prefixLen == keyLen {
currentKey = currentKey[keyLen:]
currentTrie = v
found = true
break
}
if prefixLen > 0 {
return nil, false
}
}
if !found {
break
}
}
if currentKey == "" {
return currentTrie.value, currentTrie.value != nil
}
return nil, false
}
func (t *Trie) GetLongestPrefix(key string) (string, interface{}) {
currentKey := key
currentTrie := t
matchedPrefix := ""
longestPrefix := ""
var matchedValue interface{}
for currentKey != "" {
found := false
for k, v := range currentTrie.children {
keyLen := len(k)
prefixLen := commonPrefixLength(currentKey, k)
if prefixLen == keyLen {
currentKey = currentKey[keyLen:]
currentTrie = v
matchedPrefix += k
if v.value != nil {
longestPrefix = matchedPrefix
matchedValue = v.value
}
found = true
break
}
if prefixLen > 0 {
return longestPrefix, matchedValue
}
}
if !found {
break
}
}
return longestPrefix, matchedValue
}

118
trie/trie_test.go Normal file
View File

@ -0,0 +1,118 @@
package trie
import "testing"
func TestNewTrie(t *testing.T) {
tt := NewTrie()
if len(tt.children) != 0 {
t.Log("New trie should be empty")
t.Fail()
}
if tt.value != nil {
t.Log("New trie should not have a value")
t.Fail()
}
}
func TestPutAndCount(t *testing.T) {
tt := NewTrie()
if tt.Count() != 0 {
t.Log("New trie should be empty")
t.Fail()
}
tt.Put("hello", 1)
if tt.Count() != 1 {
t.Log("One element")
t.Fail()
}
tt.Put("hello", 2)
if tt.Count() != 1 {
t.Log("Duplicate element")
t.Fail()
}
tt.Put("here", 3)
if tt.Count() != 2 {
t.Log("Two elements")
t.Fail()
}
tt.Put("there", 3)
if tt.Count() != 3 {
t.Log("Three elements")
t.Fail()
}
}
func TestPutAndGet(t *testing.T) {
tt := NewTrie()
if _, ok := tt.Get("missing"); ok {
t.Log("New trie should be empty")
t.Fail()
}
if _, ok := tt.Get(""); ok {
t.Log("New trie should be empty")
t.Fail()
}
tt.Put("hello", 1)
if r, ok := tt.Get("hello"); !ok || r != 1 {
t.Log("hello")
t.Fail()
}
tt.Put("help", 2)
if r, ok := tt.Get("help"); !ok || r != 2 {
t.Log("help")
t.Fail()
}
tt.Put("hi", 3)
if r, ok := tt.Get("hi"); !ok || r != 3 {
t.Log("hi")
t.Fail()
}
}
func TestPutAndGetLongestPrefix(t *testing.T) {
tt := NewTrie()
if _, value := tt.GetLongestPrefix(""); value != nil {
t.Log("New trie should be empty")
t.Fail()
}
tt.Put("integer", 1)
tt.Put("interval", 2)
tt.Put("in", 3)
tt.Put("string", 4)
tt.Put("struct", 5)
tt.Put("stop", 6)
if prefix, value := tt.GetLongestPrefix("interrupt"); prefix != "in" || value != 3 {
t.Log("Searching \"interrupt\"", prefix, value)
t.Fail()
}
if prefix, value := tt.GetLongestPrefix("str"); prefix != "" || value != nil {
t.Log("Searching \"str\"", prefix, value)
t.Fail()
}
if prefix, value := tt.GetLongestPrefix("struct"); prefix != "struct" || value != 5 {
t.Log("Searching \"struct\"", prefix, value)
t.Fail()
}
if prefix, value := tt.GetLongestPrefix("structure"); prefix != "struct" || value != 5 {
t.Log("Searching \"structure\"", prefix, value)
t.Fail()
}
}