Skip to content

Commit

Permalink
Ne separate dh and non-dh backends
Browse files Browse the repository at this point in the history 
Now that dhstore supports non-encrypted lookups (dhfind), there is no need to separate dhstore backends and not-dhstore backends.

This assumes there is no support for mixed dh and non-dh storage deployments.

Fixes #153
  • Loading branch information
@gammazero
gammazero committed Sep 10, 2023
1 parent 1eeef46 commit 2612b59
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 55 deletions.
10 changes: 5 additions & 5 deletions delegated_translator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ const (
unknownSchema = unknownProtocol
)

type findFunc func(ctx context.Context, method, source string, req *url.URL, body []byte, mh multihash.Multihash, encrypted bool) (int, []byte)
type findFunc func(ctx context.Context, method, source string, req *url.URL, body []byte, mh multihash.Multihash) (int, []byte)

func NewDelegatedTranslator(backend findFunc) (http.Handler, error) {
finder := delegatedTranslator{backend}
m := http.NewServeMux()
m.HandleFunc("/providers", finder.provide)
m.HandleFunc("/encrypted/providers", finder.provide)
m.HandleFunc("/providers/", func(w http.ResponseWriter, r *http.Request) { finder.find(w, r, false) })
m.HandleFunc("/encrypted/providers/", func(w http.ResponseWriter, r *http.Request) { finder.find(w, r, true) })
m.HandleFunc("/providers/", finder.find)
m.HandleFunc("/encrypted/providers/", finder.find)
return m, nil
}

Expand All @@ -61,7 +61,7 @@ func (dt *delegatedTranslator) provide(w http.ResponseWriter, r *http.Request) {
}
}

func (dt *delegatedTranslator) find(w http.ResponseWriter, r *http.Request, encrypted bool) {
func (dt *delegatedTranslator) find(w http.ResponseWriter, r *http.Request) {
_ = stats.RecordWithOptions(context.Background(),
stats.WithTags(tag.Insert(metrics.Method, r.Method)),
stats.WithMeasurements(metrics.HttpDelegatedRoutingMethod.M(1)))
Expand Down Expand Up @@ -98,7 +98,7 @@ func (dt *delegatedTranslator) find(w http.ResponseWriter, r *http.Request, encr

// Translate URL by mapping `/providers/{CID}` to `/cid/{CID}`.
uri := r.URL.JoinPath("../cid", cidUrlParam)
rcode, resp := dt.be(r.Context(), http.MethodGet, findMethodDelegated, uri, []byte{}, c.Hash(), encrypted)
rcode, resp := dt.be(r.Context(), http.MethodGet, findMethodDelegated, uri, []byte{}, c.Hash())
if rcode != http.StatusOK {
http.Error(w, "", rcode)
return
Expand Down
34 changes: 16 additions & 18 deletions find.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const (
findMethodDelegated = "delegated-v1"
)

func (s *server) findCid(w http.ResponseWriter, r *http.Request, encrypted bool) {
func (s *server) findCid(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodOptions:
discardBody(r)
Expand All @@ -39,27 +39,27 @@ func (s *server) findCid(w http.ResponseWriter, r *http.Request, encrypted bool)
discardBody(r)
http.Error(w, "invalid cid: "+err.Error(), http.StatusBadRequest)
}
s.find(w, r, c.Hash(), encrypted)
s.find(w, r, c.Hash())
default:
discardBody(r)
http.Error(w, "", http.StatusNotFound)
}
}

func (s *server) findMultihash(w http.ResponseWriter, r *http.Request, encrypted bool) {
func (s *server) findMultihash(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodOptions:
discardBody(r)
handleIPNIOptions(w, true)
case http.MethodPost:
s.find(w, r, nil, encrypted)
s.find(w, r, nil)
default:
discardBody(r)
http.Error(w, "", http.StatusNotFound)
}
}

func (s *server) findMultihashSubtree(w http.ResponseWriter, r *http.Request, encrypted bool) {
func (s *server) findMultihashSubtree(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodOptions:
discardBody(r)
Expand All @@ -71,7 +71,7 @@ func (s *server) findMultihashSubtree(w http.ResponseWriter, r *http.Request, en
discardBody(r)
http.Error(w, "invalid multihash: "+err.Error(), http.StatusBadRequest)
}
s.find(w, r, mh, encrypted)
s.find(w, r, mh)
default:
discardBody(r)
http.Error(w, "", http.StatusNotFound)
Expand All @@ -97,8 +97,8 @@ func (s *server) findMetadataSubtree(w http.ResponseWriter, r *http.Request) {

// TODO: wait for the first successful response instead
if err := sg.scatter(ctx, func(cctx context.Context, b Backend) (*[]byte, error) {
// send metadata requests only to dh backends
if _, isDhBackend := b.(dhBackend); !isDhBackend {
// Send metadata requests only to backends. This assumes
if _, isProvidersBackend := b.(providersBackend); isProvidersBackend {
return nil, nil
}

Expand Down Expand Up @@ -165,7 +165,7 @@ func (s *server) findMetadataSubtree(w http.ResponseWriter, r *http.Request) {
http.Error(w, "", http.StatusNotFound)
}

func (s *server) find(w http.ResponseWriter, r *http.Request, mh multihash.Multihash, encrypted bool) {
func (s *server) find(w http.ResponseWriter, r *http.Request, mh multihash.Multihash) {
acc, err := getAccepts(r)
if err != nil {
discardBody(r)
Expand All @@ -191,7 +191,7 @@ func (s *server) find(w http.ResponseWriter, r *http.Request, mh multihash.Multi
http.Error(w, "", http.StatusBadRequest)
return
}
rcode, resp := s.doFind(r.Context(), r.Method, findMethodOrig, r.URL, rb, mh, encrypted)
rcode, resp := s.doFind(r.Context(), r.Method, findMethodOrig, r.URL, rb, mh)
if rcode != http.StatusOK {
http.Error(w, "", rcode)
return
Expand All @@ -209,15 +209,15 @@ func (s *server) find(w http.ResponseWriter, r *http.Request, mh multihash.Multi
// JSON unless only unsupported media types are specified.
switch {
case acc.ndjson:
s.doFindNDJson(ctx, w, findMethodOrig, r.URL, false, mh, encrypted)
s.doFindNDJson(ctx, w, findMethodOrig, r.URL, false, mh)
case acc.json || acc.any || !acc.acceptHeaderFound:
if s.translateNonStreaming {
s.doFindNDJson(ctx, w, findMethodOrig, r.URL, true, mh, encrypted)
s.doFindNDJson(ctx, w, findMethodOrig, r.URL, true, mh)
return
}
// In a case where the request has no `Accept` header at all, be forgiving and respond with
// JSON.
rcode, resp := s.doFind(ctx, r.Method, findMethodOrig, r.URL, rb, mh, encrypted)
rcode, resp := s.doFind(ctx, r.Method, findMethodOrig, r.URL, rb, mh)
if rcode != http.StatusOK {
http.Error(w, "", rcode)
return
Expand All @@ -229,7 +229,7 @@ func (s *server) find(w http.ResponseWriter, r *http.Request, mh multihash.Multi
}
}

func (s *server) doFind(ctx context.Context, method, source string, req *url.URL, body []byte, mh multihash.Multihash, encrypted bool) (int, []byte) {
func (s *server) doFind(ctx context.Context, method, source string, req *url.URL, body []byte, mh multihash.Multihash) (int, []byte) {
start := time.Now()
latencyTags := []tag.Mutator{tag.Insert(metrics.Method, method)}
loadTags := []tag.Mutator{tag.Insert(metrics.Method, source)}
Expand Down Expand Up @@ -258,10 +258,8 @@ func (s *server) doFind(ctx context.Context, method, source string, req *url.URL

var count int32
if err := sg.scatter(ctx, func(cctx context.Context, b Backend) (*sgResponse, error) {
// forward double hashed requests to double hashed backends only and regular requests to regular backends
_, isDhBackend := b.(dhBackend)
_, isProvidersBackend := b.(providersBackend)
if (encrypted != isDhBackend) || isProvidersBackend {
// Forward double hashed requests to backends.
if _, isProvidersBackend := b.(providersBackend); isProvidersBackend {
return nil, nil
}

Expand Down
8 changes: 3 additions & 5 deletions find_ndjson.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func (rs *resultStats) reportMetrics(method string) {
}
}

func (s *server) doFindNDJson(ctx context.Context, w http.ResponseWriter, source string, req *url.URL, translateNonStreaming bool, mh multihash.Multihash, encrypted bool) {
func (s *server) doFindNDJson(ctx context.Context, w http.ResponseWriter, source string, req *url.URL, translateNonStreaming bool, mh multihash.Multihash) {
start := time.Now()
latencyTags := []tag.Mutator{tag.Insert(metrics.Method, http.MethodGet)}
loadTags := []tag.Mutator{tag.Insert(metrics.Method, source)}
Expand Down Expand Up @@ -170,10 +170,8 @@ func (s *server) doFindNDJson(ctx context.Context, w http.ResponseWriter, source
resultsChan := make(chan *resultWithBackend, 1)
var count int32
if err := sg.scatter(ctx, func(cctx context.Context, b Backend) (*any, error) {
// forward double hashed requests to double hashed backends only and regular requests to regular backends
_, isDhBackend := b.(dhBackend)
_, isProvidersBackend := b.(providersBackend)
if (encrypted != isDhBackend) || isProvidersBackend {
// Do not forward requests to providers backends.
if _, isProvidersBackend := b.(providersBackend); isProvidersBackend {
return nil, nil
}

Expand Down
6 changes: 1 addition & 5 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,13 @@ func main() {
},
&cli.StringSliceFlag{
Name: backendsArg,
Usage: "Backends to propagate regular requests to.",
Usage: "Backends to propagate regular and double-hashed lookup requests to.",
Value: cli.NewStringSlice("https://cid.contact/"),
},
&cli.StringSliceFlag{
Name: cascadeBackendsArg,
Usage: "Backends to propagate lookup with SERVER_CASCADE_LABELS env var as query parameter",
},
&cli.StringSliceFlag{
Name: dhBackendsArg,
Usage: "Backends to propagate Double Hashed requests to.",
},
&cli.StringSliceFlag{
Name: providersBackendsArg,
Usage: "Backends to propagate providers requests to.",
Expand Down
30 changes: 8 additions & 22 deletions server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ var (
const (
backendsArg = "backends"
cascadeBackendsArg = "cascadeBackends"
dhBackendsArg = "dhBackends"
providersBackendsArg = "providersBackends"
)

Expand All @@ -51,10 +50,6 @@ type caskadeBackend struct {
Backend
}

type dhBackend struct {
Backend
}

type providersBackend struct {
Backend
}
Expand All @@ -70,7 +65,6 @@ func NewServer(c *cli.Context) (*server, error) {
}
servers := c.StringSlice(backendsArg)
cascadeServers := c.StringSlice(cascadeBackendsArg)
dhServers := c.StringSlice(dhBackendsArg)
providersServers := c.StringSlice(providersBackendsArg)

if len(servers) == 0 {
Expand All @@ -83,7 +77,7 @@ func NewServer(c *cli.Context) (*server, error) {
}
}

backends, err := loadBackends(servers, cascadeServers, dhServers, providersServers)
backends, err := loadBackends(servers, cascadeServers, providersServers)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -150,7 +144,7 @@ func NewServer(c *cli.Context) (*server, error) {
}, nil
}

func loadBackends(servers, cascadeServers, dhServers, providersServers []string) ([]Backend, error) {
func loadBackends(servers, cascadeServers, providersServers []string) ([]Backend, error) {

newBackendFunc := func(s string) (Backend, error) {
return NewBackend(s, circuitbreaker.New(
Expand All @@ -171,13 +165,6 @@ func loadBackends(servers, cascadeServers, dhServers, providersServers []string)
}
backends = append(backends, b)
}
for _, s := range dhServers {
b, err := newBackendFunc(s)
if err != nil {
return nil, fmt.Errorf("failed to instantiate dh backend: %w", err)
}
backends = append(backends, dhBackend{Backend: b})
}
for _, s := range providersServers {
b, err := newBackendFunc(s)
if err != nil {
Expand Down Expand Up @@ -225,7 +212,6 @@ func (s *server) Reload(cctx *cli.Context) error {
}
b, err := loadBackends(surls,
cctx.StringSlice(cascadeBackendsArg),
cctx.StringSlice(dhBackendsArg),
cctx.StringSlice(providersBackendsArg))
if err != nil {
return err
Expand All @@ -237,12 +223,12 @@ func (s *server) Reload(cctx *cli.Context) error {

func (s *server) Serve() chan error {
mux := http.NewServeMux()
mux.HandleFunc("/cid/", func(w http.ResponseWriter, r *http.Request) { s.findCid(w, r, false) })
mux.HandleFunc("/encrypted/cid/", func(w http.ResponseWriter, r *http.Request) { s.findCid(w, r, true) })
mux.HandleFunc("/multihash", func(w http.ResponseWriter, r *http.Request) { s.findMultihash(w, r, false) })
mux.HandleFunc("/encrypted/multihash", func(w http.ResponseWriter, r *http.Request) { s.findMultihash(w, r, true) })
mux.HandleFunc("/multihash/", func(w http.ResponseWriter, r *http.Request) { s.findMultihashSubtree(w, r, false) })
mux.HandleFunc("/encrypted/multihash/", func(w http.ResponseWriter, r *http.Request) { s.findMultihashSubtree(w, r, true) })
mux.HandleFunc("/cid/", s.findCid)
mux.HandleFunc("/encrypted/cid/", s.findCid)
mux.HandleFunc("/multihash", s.findMultihash)
mux.HandleFunc("/encrypted/multihash", s.findMultihash)
mux.HandleFunc("/multihash/", s.findMultihashSubtree)
mux.HandleFunc("/encrypted/multihash/", s.findMultihashSubtree)
mux.HandleFunc("/metadata/", s.findMetadataSubtree)
mux.HandleFunc("/providers", s.providers)
mux.HandleFunc("/providers/", s.provider)
Expand Down

0 comments on commit 2612b59

Please sign in to comment.