commit d52389f5f2fc67aef370f8ec8ba5acfb984ccae8 Author: Maurizio Porrato Date: Sat May 8 17:03:21 2021 +0100 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f297067 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/bin/ +/data/ +/testdata/ +/.idea/ diff --git a/API.md b/API.md new file mode 100644 index 0000000..d5dc81c --- /dev/null +++ b/API.md @@ -0,0 +1,156 @@ +# API service info + +This is a Markdown conversion of the API specifications at [https://api.xbrowsersync.org/](https://api.xbrowsersync.org/) . + +## Bookmarks + +### Create bookmarks + +`POST /bookmarks` + +Creates a new (empty) bookmark sync and returns the corresponding ID. + +Post body example: + +```json +{ + "version":"1.0.0" +} +``` + +- **version:** Version number of the xBrowserSync client used to create the sync. + +Response example: + +```json +{ + "id":"52758cb942814faa9ab255208025ae59", + "lastUpdated":"2016-07-06T12:43:16.866Z", + "version":"1.0.0" +} +``` + +- **id:** 32 character alphanumeric sync ID. +- **lastUpdated:** Last updated timestamp for created bookmarks. +- **version:**Version number of the xBrowserSync client used to create the sync. + +### Get bookmarks + +`GET /bookmarks/{id}` + +Retrieves the bookmark sync corresponding to the provided sync ID. + +Query params: + +- **id:** 32 character alphanumeric sync ID. + +Response example: + +```json +{ + "bookmarks":"DWCx6wR9ggPqPRrhU4O4oLN5P09oULX4Xt+ckxswtFNds...", + "lastUpdated":"2016-07-06T12:43:16.866Z", + "version":"1.0.0" +} +``` + +- **bookmarks:** Encrypted bookmark data salted using secret value. +- **lastUpdated:** Last updated timestamp for retrieved bookmarks. +- **version:** Version number of the xBrowserSync client used to create the sync. + +### Update bookmarks + +`PUT /bookmarks/{id}` + +Updates the bookmark sync data corresponding to the provided sync ID with the provided encrypted bookmarks data. + +Query params: + +- **id:** 32 character alphanumeric sync ID. + +Post body example: + +```json +{ + "bookmarks":"DWCx6wR9ggPqPRrhU4O4oLN5P09oULX4Xt+ckxswtFNds...", + "lastUpdated":"2016-07-06T12:43:16.866Z" +} +``` + +- **bookmarks:** Encrypted bookmark data salted using secret value. +- **lastUpdated:** Last updated timestamp to check against existing bookmarks. + +Response example: + +```json +{ + "lastUpdated":"2016-07-06T12:43:16.866Z" +} +``` + +- **lastUpdated:** Last updated timestamp for updated bookmarks. + +### Get last updated + +`GET /bookmarks/{id}/lastUpdated` + +Retrieves the bookmark sync last updated timestamp corresponding to the provided sync ID. + +Query params: + +- **id:** 32 character alphanumeric sync ID. + +Response example: + +```json +{ + "lastUpdated":"2016-07-06T12:43:16.866Z" +} +``` + +- **lastUpdated:** Last updated timestamp for corresponding bookmarks. + +### Get sync version + +`GET /bookmarks/{id}/version` + +Retrieves the bookmark sync version number of the xBrowserSync client used to create the bookmarks sync corresponding +to the provided sync ID. + +Query params: + +- **id:** 32 character alphanumeric sync ID. + +Response example: + +```json +{ + "version":"1.0.0" +} +``` + +- **version:** Version number of the xBrowserSync client used to create the sync. + +## Service information + +### Get service information + +`GET /info` + +Retrieves information describing the xBrowserSync service. + +Response example: + +```json +{ + "maxSyncSize":204800, + "message":"", + "status":1, + "version":"1.0.0" +} +``` + +- **status:** Current service status code. 1 = Online; 2 = Offline; 3 = Not accepting new syncs. +- **message:** Service information message. +- **version:** API version service is using. +- **maxSyncSize:** Maximum sync size (in bytes) allowed by the service. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..c32dd18 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to \ No newline at end of file diff --git a/cmd/ubsserver/main.go b/cmd/ubsserver/main.go new file mode 100644 index 0000000..5a35df0 --- /dev/null +++ b/cmd/ubsserver/main.go @@ -0,0 +1,234 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "regexp" + "strings" + "time" + + "gitlab.com/mporrato/uBrowserSync/syncstore" +) + +type serviceInfoResp struct { + Version string `json:"version"` + Message string `json:"message"` + MaxSyncSize int32 `json:"maxSyncSize"` + Status int `json:"status"` +} + +type CreateReq struct { + Version string `json:"version"` +} + +type UpdateReq struct { + Bookmarks string `json:"bookmarks"` + LastUpdated time.Time `json:"lastUpdated"` +} + +type UpdateResp struct { + LastUpdated time.Time `json:"lastUpdated"` +} + +type LastUpdatedResp struct { + LastUpdated time.Time `json:"lastUpdated"` +} + +type GetSyncVerResp struct { + Version string `json:"version"` +} + +func sendJSON(w http.ResponseWriter, status int, data interface{}) error { + body, err := json.Marshal(data) + if err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, err = w.Write(body) + return err +} + +func sendJSONOk(w http.ResponseWriter, data interface{}) error { + return sendJSON(w, http.StatusOK, data) +} + +func sendJSONError(w http.ResponseWriter, err error) error { + switch e := err.(type) { + case syncstore.SyncError: + return sendJSON(w, e.StatusCode, e.Payload) + default: + return sendJSON(w, http.StatusInternalServerError, + syncstore.ErrorPayload{ + Code: "Internal server error", + Message: e.Error()}) + } +} + +func info(w http.ResponseWriter, req *http.Request) { + if req.Method != "GET" || req.URL.Path != "/info" { + sendJSONError(w, syncstore.NotImplementedError) + } else { + sendJSONOk(w, serviceInfoResp{ + Version: "1.1.13", + Message: "Powered by browsersync", + MaxSyncSize: 2 * 1024 * 1024, + Status: 1}) + } +} + +var store syncstore.Store + +var invalidRequestError = syncstore.NewSyncError( + "Invalid request", + "Malformed request body", + http.StatusBadRequest) + +func createSync(w http.ResponseWriter, req *http.Request) { + body := new(CreateReq) + err := json.NewDecoder(req.Body).Decode(&body) + if err != nil { + sendJSONError(w, invalidRequestError) + return + } + resp, err := store.Create(body.Version) + if err != nil { + sendJSONError(w, err) + return + } + sendJSONOk(w, resp) +} + +func getSync(syncId string, w http.ResponseWriter, req *http.Request) { + resp, err := store.Get(syncId) + if err != nil { + sendJSONError(w, err) + return + } + sendJSONOk(w, resp) +} + +func getLastUpdated(syncId string, w http.ResponseWriter, req *http.Request) { + resp, err := store.GetLastUpdated(syncId) + if err != nil { + sendJSONError(w, err) + return + } + sendJSONOk(w, LastUpdatedResp{LastUpdated: resp}) +} + +func getVersion(syncId string, w http.ResponseWriter, req *http.Request) { + resp, err := store.GetVersion(syncId) + if err != nil { + sendJSONError(w, err) + return + } + sendJSONOk(w, GetSyncVerResp{Version: resp}) +} + +func updateSync(syncId string, w http.ResponseWriter, req *http.Request) { + body := new(UpdateReq) + err := json.NewDecoder(req.Body).Decode(&body) + if err != nil { + sendJSONError(w, invalidRequestError) + return + } + resp, err := store.Update(syncId, body.Bookmarks, body.LastUpdated) + if err != nil { + sendJSONError(w, err) + return + } + sendJSONOk(w, UpdateResp{LastUpdated: resp}) +} + +var sidRe = regexp.MustCompile("^[[:xdigit:]]{32}$") + +func bookmarks(w http.ResponseWriter, req *http.Request) { + elements := strings.Split(strings.Trim(req.URL.Path, "/"), "/") + + if len(elements) == 1 && req.Method == "POST" { + createSync(w, req) + return + } + syncId := elements[1] + if !sidRe.MatchString(elements[1]) { + sendJSONError(w, syncstore.NotImplementedError) + return + } + if len(elements) == 2 { + if req.Method == "GET" { + getSync(syncId, w, req) + return + } + if req.Method == "PUT" { + updateSync(syncId, w, req) + return + } + // TODO: Handle HEAD requests + sendJSONError(w, syncstore.MethodNotImplementedError) + return + } + if len(elements) == 3 { + if elements[2] == "lastUpdated" { + if req.Method == "GET" { + getLastUpdated(syncId, w, req) + return + } + sendJSONError(w, syncstore.MethodNotImplementedError) + return + } + if elements[2] == "version" { + if req.Method == "GET" { + getVersion(syncId, w, req) + return + } + sendJSONError(w, syncstore.MethodNotImplementedError) + return + } + } + //sendJSON(w, http.StatusOK, map[string][]string{"elements": elements}) + sendJSONError(w, syncstore.NotImplementedError) +} + +var listen string + +func init() { + var err error + var storeflag string + var storeDrv syncstore.StoreDriver + + flag.StringVar(&listen, "listen", ":8090", "listen address and port") + flag.StringVar(&storeflag, "store", "fs:data", "blob store driver") + flag.Parse() + + lstore := strings.Split(storeflag, ":") + + switch lstore[0] { + case "fs": + if len(lstore) != 2 { + err = fmt.Errorf("argument required for fs store driver") + } else { + storeDrv, err = syncstore.NewFSStore(lstore[1]) + } + case "mem": + storeDrv, err = syncstore.NewMemStore() + default: + err = fmt.Errorf("Invalid store driver: "+lstore[0]) + } + if err != nil { + fmt.Println(err) + os.Exit(1) + } + store = syncstore.NewStore(storeDrv) +} + +func main() { + http.HandleFunc("/info", info) + http.HandleFunc("/bookmarks", bookmarks) + http.HandleFunc("/bookmarks/", bookmarks) + + http.ListenAndServe(listen, nil) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3a0dfe8 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gitlab.com/mporrato/uBrowserSync + +go 1.16 diff --git a/syncstore/README.md b/syncstore/README.md new file mode 100644 index 0000000..32bf861 --- /dev/null +++ b/syncstore/README.md @@ -0,0 +1,51 @@ +# uBrowserSync + +The _uncomplicated browser sync server_ is an implementation of the [xbrowsersync](https://www.xbrowsersync.org/) server API with no external dependencies. + +Being written in Go, the server is a standalone statically linked binary using the local filesystem as the datastore by +default and can be built for any operating system and hardware platform supported by the Go compiler, making it ideal +for resource constrained self-hosting scenarios. + +## Why + +xBrowserSync is a great tool for privacy minded people. The problem with the official server code is about the +software stack. It's written in NodeJS and requires MongoDB for storing data. I don't use NodeJS or MongoDB for anything +else on my VPS so adding a self-hosted xBrowserSync API server would pull in a lot of code and require a lot of +additional resources. + +The xBrowserSync API is simple enough to make writing a new server an almost trivial exercise. This was the perfect +opportunity for me to improve my Go programming skills. I'm still very far from being an expert Go developer but it was +worth my time. + +## How + +### Building + +`go install -ldflags="-s -w" -trimpath ./cmd/ubsserver` + +### Running + +`~/go/bin/ubsserver` + +TODO: Suggest deployment setups + +## Roadmap + +There are a few features missing that I would like to add. + +Hig priority: + - **Enforce storage limit**: it's missing in the current implementation but it must be implemented to prevent abuse + +Nice to have: + - **Rate limiting**: currently it can be enforced by a reverse proxy in front of the API server but would be nice to + have it as part of the server itself + - **Front page**: serving a static HTML page on `/`, perhaps with some simple JS to show the service status, should be + fairly straightforward and would improve the end user experience. The main blocker here is my lack of design skills + to craft a half decent looking front page + - **Migration tool**: a tool able to pull your current sync from a server and push it to a different one might come + handy; the main hurdle is that it would require decrypting the payload and re-encrypting it because the sync ID is + used as the IV in the AES-GCM encryption process + +## License + +The code is released under the "Unlicense" public domain license. See [UNLICENSE](UNLICENSE) for more details. \ No newline at end of file diff --git a/syncstore/blob.go b/syncstore/blob.go new file mode 100644 index 0000000..a45672a --- /dev/null +++ b/syncstore/blob.go @@ -0,0 +1,58 @@ +package syncstore + +import ( + "fmt" + "net/http" + "time" +) + +type Blob struct { + ID string `json:"id"` + Bookmarks string `json:"bookmarks"` + Version string `json:"version"` + Created time.Time `json:"created"` + LastUpdated time.Time `json:"lastUpdated"` + LastAccessed time.Time `json:"lastAccessed"` +} + +type ErrorPayload struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type SyncError struct { + StatusCode int + Payload ErrorPayload +} + +func (e SyncError) Error() string { + return fmt.Sprintf("code: %s, message: %s", e.Payload.Code, e.Payload.Message) +} + +func NewSyncError(code string, message string, status int) SyncError { + return SyncError{ + StatusCode: status, + Payload: ErrorPayload{ + Code: code, + Message: message}} +} + +var NotImplementedError = NewSyncError( + "NotImplementedException", + "The requested route has not been implemented", + http.StatusNotFound) + +var MethodNotImplementedError = NewSyncError( + "NotImplementedException", + "The requested method has not been implemented", + http.StatusMethodNotAllowed) + +var SyncNotFoundError = NewSyncError( + "SyncNotFoundException", + "Sync does not exist", + http.StatusUnauthorized) + +var SyncConflictError = NewSyncError( + "SyncConflictException", + "A sync conflict was detected", + http.StatusConflict) diff --git a/syncstore/driver_fs.go b/syncstore/driver_fs.go new file mode 100644 index 0000000..b88ceb2 --- /dev/null +++ b/syncstore/driver_fs.go @@ -0,0 +1,75 @@ +package syncstore + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +type FSStore struct { + path string +} + +func NewFSStore(path string) (*FSStore, error) { + p, err := filepath.Abs(path) + if err != nil { + return nil, err + } + err = os.MkdirAll(p, 0700) + if err != nil { + return nil, err + } + var s = new(FSStore) + s.path = p + return s, nil +} + +func (drv *FSStore) storePath(id string) string { + return filepath.Join(drv.path, id[0:3], fmt.Sprintf("%s.json", id)) +} + +func (drv *FSStore) RawSave(s *Blob) error { + body, err := json.Marshal(s) + if err != nil { + return err + } + filename := drv.storePath(s.ID) + dirname := filepath.Dir(filename) + err = os.Mkdir(dirname, 0700) + if err != nil && !os.IsExist(err) { + return err + } + f, err := os.CreateTemp(dirname, "tmp-"+s.ID+".*") + if err != nil { + return err + } + _, err = f.Write(body) + if err != nil { + _ = os.Remove(f.Name()) + return err + } + return os.Rename(f.Name(), filename) +} + +func (drv *FSStore) RawLoad(id string) (*Blob, error) { + body, err := ioutil.ReadFile(drv.storePath(id)) + if err != nil { + if os.IsNotExist(err) { + return nil, SyncNotFoundError + } + return nil, err + } + s := new(Blob) + err = json.Unmarshal(body, s) + return s, err +} + +func (drv *FSStore) Exists(id string) bool { + st, err := os.Stat(drv.storePath(id)) + if err != nil { + return false + } + return (!st.IsDir()) && (st.Size() > 0) +} diff --git a/syncstore/driver_mem.go b/syncstore/driver_mem.go new file mode 100644 index 0000000..b20ac1e --- /dev/null +++ b/syncstore/driver_mem.go @@ -0,0 +1,26 @@ +package syncstore + +type MemStore map[string]Blob + +func NewMemStore() (*MemStore, error) { + r := make(MemStore) + return &r, nil +} + +func (drv *MemStore) RawSave(s *Blob) error { + (*drv)[s.ID] = *s + return nil +} + +func (drv *MemStore) RawLoad(id string) (*Blob, error) { + r, e := (*drv)[id] + if !e { + return nil, SyncNotFoundError + } + return &r, nil +} + +func (drv *MemStore) Exists(id string) bool { + _, r := (*drv)[id] + return r +} diff --git a/syncstore/store.go b/syncstore/store.go new file mode 100644 index 0000000..fd9a858 --- /dev/null +++ b/syncstore/store.go @@ -0,0 +1,112 @@ +package syncstore + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "time" +) + +type StoreDriver interface { + RawSave(s *Blob) error + RawLoad(id string) (*Blob, error) + Exists(id string) bool +} + +type Store struct { + drv StoreDriver +} + +func NewStore(drv StoreDriver) Store { + return Store{drv} +} + +type CreateResp struct { + ID string `json:"id"` + LastUpdated time.Time `json:"lastUpdated"` + Version string `json:"version"` +} + +type GetResp struct { + Bookmarks string `json:"bookmarks"` + LastUpdated time.Time `json:"lastUpdated"` + Version string `json:"version"` +} + +func (store *Store) Create(version string) (*CreateResp, error) { + var sid string + failed := true + for i := 0; i < 5; i++ { + var idb = make([]byte, 16) + var ids = make([]byte, 32) + _, err := rand.Read(idb) + if err != nil { + return nil, err + } + hex.Encode(ids, idb) + sid = string(ids[:]) + if !store.drv.Exists(sid) { + failed = false + break + } + } + if failed { + return nil, fmt.Errorf("cannot generate unique sync ID") + } + s := new(Blob) + s.ID = sid + s.Version = version + s.Created = time.Now().UTC() + s.LastUpdated = s.Created + s.LastAccessed = s.Created + return &CreateResp{ + ID: s.ID, + LastUpdated: s.LastUpdated, + Version: s.Version}, store.drv.RawSave(s) +} + +func (store *Store) Update(id string, bookmarks string, lastUpdated time.Time) (time.Time, error) { + t, err := store.drv.RawLoad(id) + if err != nil { + return time.Time{}, err + } + if !lastUpdated.IsZero() { + if !t.LastUpdated.Equal(lastUpdated) { + return time.Time{}, SyncConflictError + } + } + t.ID = id + t.Bookmarks = bookmarks + t.LastUpdated = time.Now().UTC() + t.LastAccessed = t.LastUpdated + return t.LastUpdated, store.drv.RawSave(t) +} + +func (store *Store) Get(id string) (*GetResp, error) { + s, err := store.drv.RawLoad(id) + if err != nil { + return nil, err + } + s.ID = id + s.LastAccessed = time.Now().UTC() + return &GetResp{ + Version: s.Version, + Bookmarks: s.Bookmarks, + LastUpdated: s.LastUpdated}, store.drv.RawSave(s) +} + +func (store *Store) GetLastUpdated(id string) (time.Time, error) { + r, err := store.Get(id) + if err != nil { + return time.Time{}, err + } + return r.LastUpdated, nil +} + +func (store *Store) GetVersion(id string) (string, error) { + r, err := store.Get(id) + if err != nil { + return "", err + } + return r.Version, nil +} diff --git a/syncstore/store_test.go b/syncstore/store_test.go new file mode 100644 index 0000000..bb1a3c5 --- /dev/null +++ b/syncstore/store_test.go @@ -0,0 +1,97 @@ +package syncstore + +import ( + "regexp" + "testing" + "time" +) + +func TestStore(t *testing.T) { + drv, _ := NewMemStore() + store := NewStore(drv) + + v1 := "0.1.2" + time10 := time.Now().UTC() + t1, e1 := store.Create(v1) + time11 := time.Now().UTC() + if e1 != nil { + t.Errorf("Create() failed (%v)", e1) + } + if t1 == nil { + t.Error("Create() returned nil result") + } + idOk, err := regexp.MatchString("^[[:xdigit:]]{32}$", t1.ID) + if err != nil { + t.Errorf("Matching ID failed (%v)", err) + } + if idOk != true { + t.Errorf("Create() returned an invalid ID: %s", t1.ID) + } + if t1.Version != v1 { + t.Errorf( + "Create() returned a mismatched version: \"%s\" (expected \"%s\")", + t1.Version, v1) + } + if t1.LastUpdated.Before(time10) || t1.LastUpdated.After(time11) { + t.Errorf( + "Create() returned an incorrect lastUpdated timestamp: %v (should be between %v and %v)", + t1.LastUpdated, time10, time11) + } + t2, e2 := store.Get(t1.ID) + if e2 != nil { + t.Errorf("Get() failed (%v)", e2) + } + if t2.Bookmarks != "" { + t.Errorf("Expected empty bookmarks after Create(), got \"%s\" instead", t2.Bookmarks) + } + if t2.LastUpdated != t1.LastUpdated { + t.Errorf("lastUpdate changed from %v to %v", t1.LastUpdated, t2.LastUpdated) + } + if t2.Version != t1.Version { + t.Errorf("version changed from \"%s\" to \"%s\"", t1.Version, t2.Version) + } + testBookmark := "This is a test" + time30 := time.Now().UTC() + t3, e3 := store.Update(t1.ID, testBookmark, time.Time{}) + time31 := time.Now().UTC() + if e3 != nil { + t.Errorf("Update() with zero lastUpdated failed (%v)", e3) + } + if t3.Before(time30) || t3.After(time31) { + t.Errorf( + "Update() returned an incorrect lastUpdated timestamp: %v (should be between %v and %v)", + t3, time30, time31) + } + t4, e4 := store.Get(t1.ID) + if e4 != nil { + t.Errorf("Get() failed with error (%v)", e4) + } + if t4.Bookmarks != testBookmark { + t.Errorf("Get() did not return updated bookmarks (expected %v, got %v)", + testBookmark, t4.Bookmarks) + } + testBookmark2 := "This is another test" + _, e5 := store.Update(t1.ID, testBookmark2, t3.Add(time.Duration(-10000000))) + if e5 != SyncConflictError { + t.Errorf("Expected SyncConflictError on Update() with incorrect lastUpdated, got %v instead", e5) + } +} + +func benchCreate_helper(b *testing.B, drv StoreDriver) { + store := NewStore(drv) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + store.Create("0.1.2") + } +} + +func BenchmarkStore_Create_Mem(b *testing.B) { + drv, _ := NewMemStore() + benchCreate_helper(b, drv) +} + +func BenchmarkStore_Create_FS(b *testing.B) { + drv, _ := NewFSStore(b.TempDir()) + benchCreate_helper(b, drv) +}