diff --git a/cmd/comic-hasher/main.go b/cmd/comic-hasher/main.go new file mode 100644 index 0000000..92e511c --- /dev/null +++ b/cmd/comic-hasher/main.go @@ -0,0 +1,507 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "image" + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "io/fs" + "log" + "net/http" + _ "net/http/pprof" + "net/url" + "os" + "os/signal" + "path/filepath" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "time" + + "github.com/disintegration/imaging" + _ "golang.org/x/image/tiff" + _ "golang.org/x/image/vp8" + _ "golang.org/x/image/vp8l" + _ "golang.org/x/image/webp" + + ch "gitea.narnian.us/lordwelch/comic-hasher" + "gitea.narnian.us/lordwelch/goimagehash" + // "github.com/google/uuid" + // "github.com/zitadel/oidc/pkg/client/rp" + // httphelper "github.com/zitadel/oidc/pkg/http" + // "github.com/zitadel/oidc/pkg/oidc" +) + +type Server struct { + httpServer *http.Server + mux *http.ServeMux + BaseURL *url.URL + // token chan<- *oidc.Tokens + Partial_ahash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + Partial_dhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + Partial_phash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + Full_ahash map[uint64]ch.IDList // Maps ahash's to lists of ID's + Full_dhash map[uint64]ch.IDList // Maps dhash's to lists of ID's + Full_phash map[uint64]ch.IDList // Maps phash's to lists of ID's + // IDToCover map[string]string // IDToCover is a map of domain:ID to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0 + // covers []ch.Cover + readerQueue chan string + hashingQueue chan ch.Im + mappingQueue chan ch.Hash + // hashes are a uint64 split into 8 pieces or a unint64 for quick lookup, the value is an index to covers +} + +// var key = []byte(uuid.New().String())[:16] +var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") + +func main() { + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() + + // mustDropPrivileges() + cover_path := flag.String("cover_path", "", "path to covers to add to hash database") + flag.Parse() + if *cover_path == "" { + log.Fatal("You must supply a path") + } + st, err := os.Stat(*cover_path) + if err != nil { + panic(err) + } + fmt.Println(st) + startServer(*cover_path) +} + +func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) { + return strings.TrimSpace("lordwelch"), true +} + +// func (s *Server) setupOauthHandlers() error { +// redirectURI := *s.BaseURL +// redirectURI.Path = "/oauth/callback" +// successURI := *s.BaseURL +// successURI.Path = "/success" +// failURI := *s.BaseURL +// failURI.RawQuery = url.Values{"auth": []string{"fail"}}.Encode() + +// cookieHandler := httphelper.NewCookieHandler(key, key, httphelper.WithUnsecure()) + +// options := []rp.Option{ +// rp.WithCookieHandler(cookieHandler), +// rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)), +// } + +// provider, err := rp.NewRelyingPartyOIDC(os.Getenv("COMICHASHER_PROVIDER_URL"), os.Getenv("COMICHASHER_CLIENT_ID"), os.Getenv("COMICHASHER_CLIENT_SECRET"), redirectURI.String(), strings.Split(os.Getenv("COMICHASHER_SCOPES"), ","), options...) +// if err != nil { +// return fmt.Errorf("error creating provider: %w", err) +// } + +// // generate some state (representing the state of the user in your application, +// // e.g. the page where he was before sending him to login +// state := func() string { +// return uuid.New().String() +// } + +// // register the AuthURLHandler at your preferred path +// // the AuthURLHandler creates the auth request and redirects the user to the auth server +// // including state handling with secure cookie and the possibility to use PKCE +// s.mux.Handle("/login", rp.AuthURLHandler(state, provider)) + +// // for demonstration purposes the returned userinfo response is written as JSON object onto response +// marshalUserinfo := func(w http.ResponseWriter, r *http.Request, tokens *oidc.Tokens, state string, rp rp.RelyingParty) { +// s.token <- tokens +// w.Header().Add("location", successURI.String()) +// w.WriteHeader(301) +// } + +// // register the CodeExchangeHandler at the callbackPath +// // the CodeExchangeHandler handles the auth response, creates the token request and calls the callback function +// // with the returned tokens from the token endpoint +// s.mux.Handle(redirectURI.Path, rp.CodeExchangeHandler(marshalUserinfo, provider)) +// return nil +// } + +func (s *Server) setupAppHandlers() { + s.mux.HandleFunc("/add_cover", s.add_cover) + s.mux.HandleFunc("/get_cover", s.get_cover) + s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash) +} + +func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) { + user, authed := s.authenticated(w, r) + if !authed || user == "" { + http.Error(w, "Invalid Auth", http.StatusForbidden) + return + } + var ( + values url.Values = r.URL.Query() + domain = strings.TrimSpace(values.Get("domain")) + ID = strings.TrimSpace(values.Get("id")) + ) + if ID == "" { + log.Println("No ID Provided") + http.Error(w, "No ID Provided", http.StatusBadRequest) + return + } + if domain == "" { + log.Println("No domain Provided") + http.Error(w, "No domain Provided", http.StatusBadRequest) + return + } + // if index, ok := s.IDToCover[domain+":"+ID]; ok { + // covers, err := json.Marshal(s.covers[index]) + // if err == nil { + // w.Header().Add("Content-Type", "application/json") + // w.Write(covers) + // return + // } + // } + fmt.Fprintln(w, "Not implemented") +} + +func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result { + + var foundMatches []ch.Result + + if matchedResults, ok := s.Full_ahash[ahash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.AHash}}) + } + if matchedResults, ok := s.Full_dhash[dhash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.DHash}}) + } + if matchedResults, ok := s.Full_phash[phash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.PHash}}) + } + + // If we have exact matches don't bother with other matches + if len(foundMatches) > 0 { + return foundMatches + } + + for i, partial_hash := range ch.SplitHash(ahash) { + for _, match := range ch.Atleast(8, ahash, s.Partial_ahash[i][partial_hash]) { + if matchedResults, ok := s.Full_ahash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.AHash}}) + } + } + } + + for i, partial_hash := range ch.SplitHash(dhash) { + for _, match := range ch.Atleast(8, dhash, s.Partial_dhash[i][partial_hash]) { + if matchedResults, ok := s.Full_dhash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.DHash}}) + } + } + } + + for i, partial_hash := range ch.SplitHash(phash) { + for _, match := range ch.Atleast(8, phash, s.Partial_phash[i][partial_hash]) { + if matchedResults, ok := s.Full_phash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.PHash}}) + } + } + } + + return foundMatches +} + +func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) { + user, authed := s.authenticated(w, r) + if !authed || user == "" { + http.Error(w, "Invalid Auth", http.StatusForbidden) + return + } + var ( + values = r.URL.Query() + ahashStr = strings.TrimSpace(values.Get("ahash")) + dhashStr = strings.TrimSpace(values.Get("dhash")) + phashStr = strings.TrimSpace(values.Get("phash")) + ahash uint64 + dhash uint64 + phash uint64 + err error + ) + if ahash, err = strconv.ParseUint(ahashStr, 16, 64); err != nil && ahashStr != "" { + log.Printf("could not parse ahash: %s", ahashStr) + http.Error(w, "parse fail", http.StatusBadRequest) + return + } + if dhash, err = strconv.ParseUint(dhashStr, 16, 64); err != nil && dhashStr != "" { + log.Printf("could not parse dhash: %s", dhashStr) + http.Error(w, "parse fail", http.StatusBadRequest) + return + } + if phash, err = strconv.ParseUint(phashStr, 16, 64); err != nil && phashStr != "" { + log.Printf("could not parse phash: %s", phashStr) + http.Error(w, "parse fail", http.StatusBadRequest) + return + } + matches := s.getMatches(ahash, dhash, phash) + if len(matches) > 0 { + covers, err := json.Marshal(matches) + log.Println(err) + w.Header().Add("Content-Type", "application/json") + w.Write(covers) + w.Write([]byte{'\n'}) + return + } + + w.Header().Add("Content-Type", "application/json") + fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}") +} + +func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) { + user, authed := s.authenticated(w, r) + if !authed || user == "" { + http.Error(w, "Invalid Auth", http.StatusForbidden) + return + } + var ( + values = r.URL.Query() + domain = strings.TrimSpace(values.Get("domain")) + ID = strings.TrimSpace(values.Get("id")) + ) + if ID == "" { + log.Println("No ID Provided") + http.Error(w, "No ID Provided", http.StatusBadRequest) + return + } + if domain == "" { + log.Println("No domain Provided") + http.Error(w, "No domain Provided", http.StatusBadRequest) + return + } + i, format, err := image.Decode(r.Body) + if err != nil { + msg := fmt.Sprintf("Failed to decode Image: %s", err) + log.Println(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + log.Printf("Decoded %s image from %s", format, user) + s.hashingQueue <- ch.Im{Im: i, Format: format, Domain: ch.Source(domain), ID: ID, Path: ""} + fmt.Fprintln(w, "Success") +} + +func (s *Server) mapHashes(hash ch.Hash) { + _, ahash_ok := s.Full_ahash[hash.Ahash.GetHash()] + if !ahash_ok { + s.Full_ahash[hash.Ahash.GetHash()] = make(ch.IDList) + } + s.Full_ahash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.Full_ahash[hash.Ahash.GetHash()][hash.Domain], hash.ID) + _, dhash_ok := s.Full_dhash[hash.Dhash.GetHash()] + if !dhash_ok { + s.Full_dhash[hash.Dhash.GetHash()] = make(ch.IDList) + } + s.Full_dhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.Full_dhash[hash.Dhash.GetHash()][hash.Domain], hash.ID) + _, phash_ok := s.Full_phash[hash.Phash.GetHash()] + if !phash_ok { + s.Full_phash[hash.Phash.GetHash()] = make(ch.IDList) + } + s.Full_phash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.Full_phash[hash.Phash.GetHash()][hash.Domain], hash.ID) + + for i, partial_hash := range ch.SplitHash(hash.Ahash.GetHash()) { + s.Partial_ahash[i][partial_hash] = ch.Insert(s.Partial_ahash[i][partial_hash], hash.Ahash.GetHash()) + } + for i, partial_hash := range ch.SplitHash(hash.Dhash.GetHash()) { + s.Partial_dhash[i][partial_hash] = ch.Insert(s.Partial_dhash[i][partial_hash], hash.Dhash.GetHash()) + } + for i, partial_hash := range ch.SplitHash(hash.Phash.GetHash()) { + s.Partial_phash[i][partial_hash] = ch.Insert(s.Partial_phash[i][partial_hash], hash.Phash.GetHash()) + } +} + +func (s *Server) initHashes() { + for i := range s.Partial_ahash { + s.Partial_ahash[i] = make(map[uint8][]uint64) + } + for i := range s.Partial_dhash { + s.Partial_dhash[i] = make(map[uint8][]uint64) + } + for i := range s.Partial_phash { + s.Partial_phash[i] = make(map[uint8][]uint64) + } + s.Full_ahash = make(map[uint64]ch.IDList) + s.Full_dhash = make(map[uint64]ch.IDList) + s.Full_phash = make(map[uint64]ch.IDList) + // s.IDToCover = make(map[string]string) +} + +func (s *Server) mapper() { + var total uint64 = 0 + for { + select { + case hash := <-s.mappingQueue: + if total%1000 == 0 { + mem := ch.MemStats() + if mem > 10*1024*1024*1024 { + fmt.Println("Forcing gc", mem, "G") + runtime.GC() + } + } + total++ + + s.mapHashes(hash) + } + } +} + +func (s *Server) hasher(workerID int) { + for { + select { + case i := <-s.hashingQueue: + start := time.Now() + + hash := ch.HashImage(i) + if hash.Domain == "" { + continue + } + + s.mappingQueue <- hash + + elapsed := time.Now().Sub(start) + // fmt.Printf("%#064b\n", ahash.GetHash()) + // fmt.Printf("%#064b\n", dhash.GetHash()) + // fmt.Printf("%#064b\n", phash.GetHash()) + log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, i.Path, hash.Ahash.GetHash(), hash.ID) + } + } +} + +func (s *Server) reader(workerID int) { + for { + select { + case path := <-s.readerQueue: + file, err := os.Open(path) + if err != nil { + panic(err) + } + i, format, err := image.Decode(bufio.NewReader(file)) + if err != nil { + continue // skip this image + } + file.Close() + // fmt.Println("Hashing", path) + im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path} + s.hashingQueue <- im + } + } +} + +// func (s *Server) CoverByID(ID string) uint32 { +// v,ok :=s.IDToCover[ID] +// return 0 +// } +func (s *Server) FindHashes() { +} + +func startServer(cover_path string) { + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + mux := http.NewServeMux() + server := Server{ + // token: make(chan *oidc.Tokens), + readerQueue: make(chan string, 1120130), // Number gotten from checking queue size + hashingQueue: make(chan ch.Im), + mappingQueue: make(chan ch.Hash), + mux: mux, + httpServer: &http.Server{ + Addr: ":8080", + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + }, + } + imaging.SetMaxProcs(1) + fmt.Println("init hashes") + server.initHashes() + // server.setupOauthHandlers() + fmt.Println("init handlers") + server.setupAppHandlers() + fmt.Println("init hashers") + go server.reader(1) + go server.reader(2) + go server.reader(3) + go server.reader(4) + go server.reader(5) + go server.reader(6) + go server.reader(7) + go server.reader(8) + go server.reader(9) + go server.reader(10) + + go server.hasher(1) + go server.hasher(2) + go server.hasher(3) + go server.hasher(4) + go server.hasher(5) + go server.hasher(6) + go server.hasher(7) + go server.hasher(8) + go server.hasher(9) + go server.hasher(10) + + fmt.Println("init mapper") + go server.mapper() + + fmt.Println("Starting local hashing go routine") + go func() { + fmt.Println("Hashing covers at ", cover_path) + start := time.Now() + err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error { + select { + case s := <-sig: + server.httpServer.Shutdown(context.TODO()) + return fmt.Errorf("Signal: %v", s) + default: + } + if d.IsDir() { // Only hash thumbnails for now + return nil + } + fmt.Println(len(server.readerQueue)) + server.readerQueue <- path + return nil + }) + elapsed := time.Now().Sub(start) + fmt.Println("Err:", err, "local hashing took", elapsed) + + select { + case s := <-sig: + server.httpServer.Shutdown(context.TODO()) + log.Printf("Signal: %v", s) + } + }() + + fmt.Println("Listening on ", server.httpServer.Addr) + err := server.httpServer.ListenAndServe() + if err != nil { + fmt.Println(err) + } + f, er := os.Create("memprofile") + if er != nil { + fmt.Println("Error in creating file for writing memory profile to: ", er) + return + } + defer f.Close() + runtime.GC() + if e := pprof.WriteHeapProfile(f); e != nil { + fmt.Println("Error in writing memory profile: ", e) + return + } +} diff --git a/cmd/hash.py b/cmd/hash.py index 16a58b4..5a32177 100644 --- a/cmd/hash.py +++ b/cmd/hash.py @@ -1,10 +1,11 @@ from typing import Collection, Sequence from PIL import Image -import argparse,pathlib,numpy,imagehash +import argparse,pathlib,numpy,imagehash,sys ap = argparse.ArgumentParser() ap.add_argument("--file", type=pathlib.Path) +ap.add_argument("--debug", action='store_true') opts = ap.parse_args() opts.file = pathlib.Path(opts.file) @@ -18,39 +19,42 @@ resized = gray.copy().resize((hash_size, hash_size), Image.Resampling.LANCZOS) def print_image(image: Image.Image) -> None: for row in numpy.asarray(image): - print('[ ', end='') + print('[ ', end='', file=sys.stderr) for i in row: if isinstance(i, Collection): - print('{ ', end='') + print('{ ', end='', file=sys.stderr) for idx, x in enumerate(i): if idx == len(i)-1: - print(f'{int(x):03d} ', end='') + print(f'{int(x):03d} ', end='', file=sys.stderr) else: - print(f'{int(x):03d}, ', end='') - print('}, ', end='') + print(f'{int(x):03d}, ', end='', file=sys.stderr) + print('}, ', end='', file=sys.stderr) else: - print(f'{int(i):03d}, ', end='') - print(']') + print(f'{int(i):03d}, ', end='', file=sys.stderr) + print(']', file=sys.stderr) def bin_str(hash): return ''.join(str(b) for b in 1 * hash.hash.flatten()) -# print("rgb") -# print_image(image) -# print() -image.save("py.rgb.png") +if opts.debug: + image.save("py.rgb.png") + print("rgb", file=sys.stderr) + print_image(image) + print(file=sys.stderr) -# print("gray") -# print_image(gray) -gray.save("py.gray.png") -# print() +if opts.debug: + gray.save("py.gray.png") + print("gray", file=sys.stderr) + print_image(gray) + print(file=sys.stderr) -# print("resized") -# print_image(resized) -resized.save("py.resized.png") -# print() +if opts.debug: + resized.save("py.resized.png") + print("resized", file=sys.stderr) + print_image(resized) + print(file=sys.stderr) -print(str(imagehash.average_hash(image))) -print(str(imagehash.dhash(image))) -print(str(imagehash.phash(image))) +print('ahash: ', bin_str(imagehash.average_hash(image))) +print('dhash: ', bin_str(imagehash.dhash(image))) +print('phash: ', bin_str(imagehash.phash(image))) diff --git a/cmd/hash/main.go b/cmd/hash/main.go index 1bb1973..ff452b4 100644 --- a/cmd/hash/main.go +++ b/cmd/hash/main.go @@ -1,28 +1,20 @@ package main import ( + "bufio" "flag" "fmt" "image" - "image/draw" - _ "image/gif" _ "image/jpeg" - - // "github.com/pixiv/go-libjpeg/jpeg" "image/png" "log" "os" "strings" - "gitea.narnian.us/lordwelch/goimagehash" - "gitea.narnian.us/lordwelch/goimagehash/transforms" - "github.com/anthonynsimon/bild/transform" - ih "gitea.narnian.us/lordwelch/image-hasher" - _ "github.com/gen2brain/avif" - _ "github.com/spakin/netpbm" - _ "golang.org/x/image/bmp" - _ "golang.org/x/image/tiff" _ "golang.org/x/image/webp" + + ch "gitea.narnian.us/lordwelch/comic-hasher" + "gitea.narnian.us/lordwelch/goimagehash" ) func init() { @@ -34,27 +26,6 @@ func init() { } -func ToGray(img image.Image, pix []uint8) *image.Gray { - c := img.Bounds().Dx() * img.Bounds().Dy() - if cap(pix) < c { - pix = append([]byte(nil), make([]byte, c)...) - } - pix = pix[:c] - gray := &image.Gray{ - Pix: transforms.Rgb2Gray(img, pix), - Stride: img.Bounds().Dx(), - Rect: img.Bounds(), - } - return gray -} - -func resize(img image.Image, w, h int) *image.Gray { - resized := transform.Resize(img, w, h, transform.Lanczos) - r_gray := image.NewGray(image.Rect(0, 0, resized.Bounds().Dx(), resized.Bounds().Dy())) - draw.Draw(r_gray, resized.Bounds(), resized, resized.Bounds().Min, draw.Src) - return r_gray -} - func save_image(im image.Image, name string) { file, err := os.Create(name) if err != nil { @@ -89,22 +60,27 @@ func fmtImage(im image.Image) string { } func debugImage(im image.Image, width, height int) { - gray := ToGray(im, nil) - resized := resize(gray, width, height) + gray := goimagehash.ToGray(im, nil) + resized := goimagehash.Resize(gray, width, height, nil) + save_image(im, "go.rgb.png") log.Println("rgb") log.Println(fmtImage(im)) - save_image(im, "go.rgb.png") + + save_image(gray, "go.gray.png") log.Println("gray") log.Println(fmtImage(gray)) - save_image(gray, "go.gray.png") + + save_image(resized, "go.resized.png") log.Println("resized") log.Println(fmtImage(resized)) - save_image(resized, "go.resized.png") + } func main() { + log.SetFlags(0) imPath := flag.String("file", "", "image file to hash") + debug := flag.Bool("debug", false, "Enable debug output") flag.Parse() if imPath == nil || *imPath == "" { flag.Usage() @@ -117,47 +93,24 @@ func main() { return } defer file.Close() - im, format, err := image.Decode(file) + im, format, err := image.Decode(bufio.NewReader(file)) if err != nil { msg := fmt.Sprintf("Failed to decode Image: %s", err) log.Println(msg) return } - + debugim := im if format == "webp" { - im = goimagehash.FancyUpscale(im.(*image.YCbCr)) + debugim = goimagehash.FancyUpscale(im.(*image.YCbCr)) } - debugImage(im, 8, 8) - - var ( - ahash *goimagehash.ImageHash - dhash *goimagehash.ImageHash - phash *goimagehash.ImageHash - ) - - ahash, err = goimagehash.AverageHash(im) - if err != nil { - msg := fmt.Sprintf("Failed to ahash Image: %s", err) - log.Println(msg) - return + if *debug { + debugImage(debugim, 8, 8) } - dhash, err = goimagehash.DifferenceHash(im) - if err != nil { - msg := fmt.Sprintf("Failed to ahash Image: %s", err) - log.Println(msg) - return - } + hash := ch.HashImage(ch.Im{Im: im, Format: format, Domain: ch.Source("comicvine.gamespot.com"), ID: "nothing"}) - phash, err = goimagehash.PerceptionHash(im) - if err != nil { - msg := fmt.Sprintf("Failed to ahash Image: %s", err) - log.Println(msg) - return - } - - fmt.Println("ahash: ", ahash.BinString()) - fmt.Println("dhash: ", dhash.BinString()) - fmt.Println("phash: ", phash.BinString()) + fmt.Println("ahash: ", hash.Ahash.BinString()) + fmt.Println("dhash: ", hash.Dhash.BinString()) + fmt.Println("phash: ", hash.Phash.BinString()) } diff --git a/go.mod b/go.mod index 4c45efc..e443af2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitea.narnian.us/lordwelch/image-hasher +module gitea.narnian.us/lordwelch/comic-hasher go 1.22.1 @@ -6,13 +6,9 @@ toolchain go1.22.2 require ( gitea.narnian.us/lordwelch/goimagehash v0.0.0-20240502010648-cb5a8237c420 - github.com/anthonynsimon/bild v0.13.0 + github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09 github.com/fmartingr/go-comicinfo/v2 v2.0.2 - github.com/gen2brain/avif v0.3.1 - github.com/google/uuid v1.3.0 github.com/mholt/archiver/v4 v4.0.0-alpha.8 - github.com/spakin/netpbm v1.3.0 - github.com/zitadel/oidc v1.13.4 golang.org/x/image v0.7.0 ) @@ -23,7 +19,7 @@ require ( github.com/bodgit/windows v1.0.0 // indirect github.com/connesc/cipherio v0.2.1 // indirect github.com/dsnet/compress v0.0.1 // indirect - github.com/ebitengine/purego v0.7.1 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -31,26 +27,13 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect - github.com/tetratelabs/wazero v1.7.1 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/sys v0.19.0 // indirect ) -require ( - github.com/golang/protobuf v1.5.3 // indirect - github.com/gorilla/schema v1.2.0 // indirect - github.com/gorilla/securecookie v1.1.1 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/text v0.14.0 - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect -) +require golang.org/x/text v0.14.0 replace golang.org/x/text v0.14.0 => /home/timmy/build/source/text/ diff --git a/go.sum b/go.sum index eac1e0a..0921230 100644 --- a/go.sum +++ b/go.sum @@ -19,9 +19,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/anthonynsimon/bild v0.13.0 h1:mN3tMaNds1wBWi1BrJq0ipDBhpkooYfu7ZFSMhXt1C8= -github.com/anthonynsimon/bild v0.13.0/go.mod h1:tpzzp0aYkAsMi1zmfhimaDyX1xjn2OUc1AJZK/TF0AE= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bodgit/plumbing v1.2.0 h1:gg4haxoKphLjml+tgnecR4yLBV5zo4HAZGCtAh3xCzM= github.com/bodgit/plumbing v1.2.0/go.mod h1:b9TeRi7Hvc6Y05rjm8VML3+47n4XTZPtQ/5ghqic2n8= github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY= @@ -35,25 +32,18 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw= github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09 h1:MJFqtdxTq94XqUgg7DcGCaOIXrDTJE/tPHK66Jshguc= +github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA= -github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fmartingr/go-comicinfo/v2 v2.0.2 h1:VppvrHr8C4+iktBTOd7vzTMNbVecZ7F/Ji1kPTOIGg4= github.com/fmartingr/go-comicinfo/v2 v2.0.2/go.mod h1:LUu/VenzEJkJt2PN49Kfpe50IgZkVkvH0m9Fnld8Dh0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gen2brain/avif v0.3.1 h1:womS2LKvhS/dSR3zIKUxtJW+riGlY48akGWqc+YgHtE= -github.com/gen2brain/avif v0.3.1/go.mod h1:s9sI2zo2cF6EdyRVCtnIfwL/Qb3k0TkOIEsz6ovK1ms= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -71,9 +61,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -82,35 +69,20 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= -github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jeremija/gosubmit v0.2.7 h1:At0OhGCFGPXyjPYAsCchoBUhE099pcBXmsb4iZqROIc= -github.com/jeremija/gosubmit v0.2.7/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -123,68 +95,40 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= -github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spakin/netpbm v1.3.0 h1:eDX7VvrkN5sHXW0luZXRA4AKDlLmu0E5sNxJ7VSTwxc= -github.com/spakin/netpbm v1.3.0/go.mod h1:Q+ep6vNv1G44qSWp0wt3Y9o1m/QXjmaXZIFC0PMVpq0= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8= -github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zitadel/logging v0.3.4 h1:9hZsTjMMTE3X2LUi0xcF9Q9EdLo+FAezeu52ireBbHM= -github.com/zitadel/logging v0.3.4/go.mod h1:aPpLQhE+v6ocNK0TWrBrd363hZ95KcI17Q1ixAQwZF0= -github.com/zitadel/oidc v1.13.4 h1:+k2GKqP9Ld9S2MSFlj+KaNsoZ3J9oy+Ezw51EzSFuC8= -github.com/zitadel/oidc v1.13.4/go.mod h1:3h2DhUcP02YV6q/CA/BG4yla0o6rXjK+DkJGK/dwJfw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -196,8 +140,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw= golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -215,6 +159,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -232,27 +177,24 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -266,12 +208,12 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -309,11 +251,13 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -327,8 +271,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -349,15 +291,9 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 959aa04..30e4ab3 100644 --- a/main.go +++ b/main.go @@ -1,380 +1,100 @@ -package main +package ch import ( - "bufio" - "context" - "encoding/json" - "flag" + "cmp" "fmt" "image" - _ "image/gif" - _ "image/jpeg" - _ "image/png" - "io/fs" "log" - "net/http" - "net/url" - "os" - "os/signal" - "path/filepath" + "math/bits" "runtime" - "runtime/pprof" - "strconv" - "strings" - "time" - - _ "golang.org/x/image/tiff" - _ "golang.org/x/image/vp8" - _ "golang.org/x/image/vp8l" - _ "golang.org/x/image/webp" + "slices" "gitea.narnian.us/lordwelch/goimagehash" - "github.com/google/uuid" - - "github.com/zitadel/oidc/pkg/client/rp" - httphelper "github.com/zitadel/oidc/pkg/http" - "github.com/zitadel/oidc/pkg/oidc" ) const ( - h_1 uint64 = 0b11111111 << (8 * iota) - h_2 - h_3 - h_4 - h_5 - h_6 - h_7 - h_8 + H_0 uint64 = 0b11111111 << (8 * iota) + H_1 + H_2 + H_3 + H_4 + H_5 + H_6 + H_7 ) const ( - shift_1 = (8 * iota) - shift_2 - shift_3 - shift_4 - shift_5 - shift_6 - shift_7 - shift_8 + Shift_0 = (8 * iota) + Shift_1 + Shift_2 + Shift_3 + Shift_4 + Shift_5 + Shift_6 + Shift_7 ) -type Cover map[string][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"} +type Source string -// type Cover struct { -// AHash uint64 -// DHash uint64 -// PHash uint64 -// IDs map[string][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"} -// } - -type Server struct { - httpServer *http.Server - mux *http.ServeMux - BaseURL *url.URL - token chan<- *oidc.Tokens - ahash [8]map[uint8]string - dhash [8]map[uint8]string - phash [8]map[uint8]string - fAhash map[uint64]string - fDhash map[uint64]string - fPhash map[uint64]string - IDToCover map[string]string // IDToCover is a map of domain:id to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0 - covers []Cover - hashingQueue chan im - mappingQueue chan hash - // hashes are a uint64 split into 8 pieces or a unint64 for quick lookup, the value is an index to covers +type Match struct { + Distance int + Hash uint64 } -type im struct { - im image.Image - format string - domain string - id string +type Result struct { + IDs IDList + Distance int + Hash ImageHash } -type hash struct { - ahash *goimagehash.ImageHash - dhash *goimagehash.ImageHash - phash *goimagehash.ImageHash - domain string - id string +type Im struct { + Im image.Image + Format string + Domain Source + ID, Path string } -var key = []byte(uuid.New().String())[:16] - -func main() { - // mustDropPrivileges() - cover_path := flag.String("cover_path", "", "path to covers to add to hash database") - flag.Parse() - if *cover_path == "" { - log.Fatal("You must supply a path") - } - st, err := os.Stat(*cover_path) - if err != nil { - panic(err) - } - fmt.Println(st) - startServer(*cover_path) +type Hash struct { + Ahash *goimagehash.ImageHash + Dhash *goimagehash.ImageHash + Phash *goimagehash.ImageHash + Domain Source + ID string } -func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) { - return strings.TrimSpace("lordwelch"), true +type ImageHash struct { + Hash uint64 + Kind goimagehash.Kind } -func (s *Server) setupOauthHandlers() error { - redirectURI := *s.BaseURL - redirectURI.Path = "/oauth/callback" - successURI := *s.BaseURL - successURI.Path = "/success" - failURI := *s.BaseURL - failURI.RawQuery = url.Values{"auth": []string{"fail"}}.Encode() - - cookieHandler := httphelper.NewCookieHandler(key, key, httphelper.WithUnsecure()) - - options := []rp.Option{ - rp.WithCookieHandler(cookieHandler), - rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)), - } - - provider, err := rp.NewRelyingPartyOIDC(os.Getenv("COMICHASHER_PROVIDER_URL"), os.Getenv("COMICHASHER_CLIENT_ID"), os.Getenv("COMICHASHER_CLIENT_SECRET"), redirectURI.String(), strings.Split(os.Getenv("COMICHASHER_SCOPES"), ","), options...) - if err != nil { - return fmt.Errorf("error creating provider: %w", err) - } - - // generate some state (representing the state of the user in your application, - // e.g. the page where he was before sending him to login - state := func() string { - return uuid.New().String() - } - - // register the AuthURLHandler at your preferred path - // the AuthURLHandler creates the auth request and redirects the user to the auth server - // including state handling with secure cookie and the possibility to use PKCE - s.mux.Handle("/login", rp.AuthURLHandler(state, provider)) - - // for demonstration purposes the returned userinfo response is written as JSON object onto response - marshalUserinfo := func(w http.ResponseWriter, r *http.Request, tokens *oidc.Tokens, state string, rp rp.RelyingParty) { - s.token <- tokens - w.Header().Add("location", successURI.String()) - w.WriteHeader(301) - } - - // register the CodeExchangeHandler at the callbackPath - // the CodeExchangeHandler handles the auth response, creates the token request and calls the callback function - // with the returned tokens from the token endpoint - s.mux.Handle(redirectURI.Path, rp.CodeExchangeHandler(marshalUserinfo, provider)) - return nil -} - -func (s *Server) setupAppHandlers() { - s.mux.HandleFunc("/add_cover", s.add_cover) - s.mux.HandleFunc("/get_cover", s.get_cover) - s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash) -} - -func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) { - user, authed := s.authenticated(w, r) - if !authed || user == "" { - http.Error(w, "Invalid Auth", http.StatusForbidden) - return - } - var ( - values = r.URL.Query() - domain = strings.TrimSpace(values.Get("domain")) - id = strings.TrimSpace(values.Get("id")) - ) - if id == "" { - log.Println("No ID Provided") - http.Error(w, "No ID Provided", http.StatusBadRequest) - return - } - if domain == "" { - log.Println("No domain Provided") - http.Error(w, "No domain Provided", http.StatusBadRequest) - return - } - // if index, ok := s.IDToCover[domain+":"+id]; ok { - // covers, err := json.Marshal(s.covers[index]) - // if err == nil { - // w.Header().Add("Content-Type", "application/json") - // w.Write(covers) - // return - // } - // } - fmt.Fprintln(w, "Not implemented") -} - -func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) { - user, authed := s.authenticated(w, r) - if !authed || user == "" { - http.Error(w, "Invalid Auth", http.StatusForbidden) - return - } - var ( - values = r.URL.Query() - ahashStr = strings.TrimSpace(values.Get("ahash")) - dhashStr = strings.TrimSpace(values.Get("dhash")) - phashStr = strings.TrimSpace(values.Get("phash")) - ahash uint64 - dhash uint64 - phash uint64 - err error - ) - if ahash, err = strconv.ParseUint(ahashStr, 16, 64); err != nil && ahashStr != "" { - log.Printf("could not parse ahash: %s", ahashStr) - http.Error(w, "parse fail", http.StatusBadRequest) - return - } - if dhash, err = strconv.ParseUint(dhashStr, 16, 64); err != nil && dhashStr != "" { - log.Printf("could not parse dhash: %s", dhashStr) - http.Error(w, "parse fail", http.StatusBadRequest) - return - } - if phash, err = strconv.ParseUint(phashStr, 16, 64); err != nil && phashStr != "" { - log.Printf("could not parse phash: %s", phashStr) - http.Error(w, "parse fail", http.StatusBadRequest) - return - } - type res struct { - Domain string - Id string - } - if index, ok := s.fAhash[ahash]; ok { - covers, err := json.Marshal(res{ - strings.Split(index, ":")[0], - strings.Split(index, ":")[1], - }) - if err == nil { - w.Header().Add("Content-Type", "application/json") - w.Write(covers) - w.Write([]byte{'\n'}) - return - } else { - fmt.Println(err) +func Atleast(maxDistance int, search_hash uint64, hashes []uint64) []Match { + matching_hashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them + for _, stored_hash := range hashes { + distance := bits.OnesCount64(search_hash ^ stored_hash) + if distance <= maxDistance { + matching_hashes = append(matching_hashes, Match{distance, stored_hash}) } } - if index, ok := s.fDhash[dhash]; ok { - covers, err := json.Marshal(res{ - strings.Split(index, ":")[0], - strings.Split(index, ":")[1], - }) - if err == nil { - w.Header().Add("Content-Type", "application/json") - w.Write(covers) - w.Write([]byte{'\n'}) - return - } else { - fmt.Println(err) - } - } - if index, ok := s.fPhash[phash]; ok { - covers, err := json.Marshal(res{ - strings.Split(index, ":")[0], - strings.Split(index, ":")[1], - }) - if err == nil { - w.Header().Add("Content-Type", "application/json") - w.Write(covers) - w.Write([]byte{'\n'}) - return - } else { - fmt.Println(err) - } - } - w.Header().Add("Content-Type", "application/json") - fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}") + return matching_hashes } -func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) { - user, authed := s.authenticated(w, r) - if !authed || user == "" { - http.Error(w, "Invalid Auth", http.StatusForbidden) - return +func Insert[S ~[]E, E cmp.Ordered](slice S, item E) S { + index, item_found := slices.BinarySearch(slice, item) + if item_found { + return slice } - var ( - values = r.URL.Query() - domain = strings.TrimSpace(values.Get("domain")) - id = strings.TrimSpace(values.Get("id")) - ) - if id == "" { - log.Println("No ID Provided") - http.Error(w, "No ID Provided", http.StatusBadRequest) - return - } - if domain == "" { - log.Println("No domain Provided") - http.Error(w, "No domain Provided", http.StatusBadRequest) - return - } - i, format, err := image.Decode(r.Body) - if err != nil { - msg := fmt.Sprintf("Failed to decode Image: %s", err) - log.Println(msg) - http.Error(w, msg, http.StatusBadRequest) - return - } - log.Printf("Decoded %s image from %s", format, user) - s.hashingQueue <- im{i, format, domain, id} - fmt.Fprintln(w, "Success") + return slices.Insert(slice, index, item) } -func (s *Server) mapHashes(id string, ahash, dhash, phash *goimagehash.ImageHash) { - s.fAhash[ahash.GetHash()] = id - s.fDhash[dhash.GetHash()] = id - s.fPhash[phash.GetHash()] = id - for i, partial_hash := range SplitHash(ahash.GetHash()) { - s.ahash[i][partial_hash] = id - } - for i, partial_hash := range SplitHash(dhash.GetHash()) { - s.dhash[i][partial_hash] = id - } - for i, partial_hash := range SplitHash(phash.GetHash()) { - s.phash[i][partial_hash] = id - } -} - -func (s *Server) initHashes() { - for i := range s.ahash { - s.ahash[i] = make(map[uint8]string) - } - for i := range s.dhash { - s.dhash[i] = make(map[uint8]string) - } - for i := range s.phash { - s.phash[i] = make(map[uint8]string) - } - s.fAhash = make(map[uint64]string) - s.fDhash = make(map[uint64]string) - s.fPhash = make(map[uint64]string) - s.IDToCover = make(map[string]string) -} func MemStats() uint64 { var m runtime.MemStats runtime.ReadMemStats(&m) return m.Alloc } -func (s *Server) mapper() { - var total uint64 = 0 - for { - select { - case hash := <-s.mappingQueue: - if total%1000 == 0 { - mem := MemStats() - if mem > 10*1024*1024*1024 { - fmt.Println("Forcing gc", mem, "G") - runtime.GC() - } - } - total++ - s.mapHashes(hash.domain+":"+hash.id, hash.ahash, hash.dhash, hash.phash) - } - } -} - -func hashImage(i im) hash { - if i.format == "webp" { - i.im = goimagehash.FancyUpscale(i.im.(*image.YCbCr)) +func HashImage(i Im) Hash { + if i.Format == "webp" { + i.Im = goimagehash.FancyUpscale(i.Im.(*image.YCbCr)) } var ( @@ -384,156 +104,44 @@ func hashImage(i im) hash { phash *goimagehash.ImageHash ) - ahash, err = goimagehash.AverageHash(i.im) + ahash, err = goimagehash.AverageHash(i.Im) if err != nil { msg := fmt.Sprintf("Failed to ahash Image: %s", err) log.Println(msg) - return hash{} + return Hash{} } - dhash, err = goimagehash.DifferenceHash(i.im) + dhash, err = goimagehash.DifferenceHash(i.Im) if err != nil { msg := fmt.Sprintf("Failed to dhash Image: %s", err) log.Println(msg) - return hash{} + return Hash{} } - phash, err = goimagehash.PerceptionHash(i.im) + phash, err = goimagehash.PerceptionHash(i.Im) if err != nil { msg := fmt.Sprintf("Failed to phash Image: %s", err) log.Println(msg) - return hash{} + return Hash{} } - return hash{ - ahash: ahash, - dhash: dhash, - phash: phash, - domain: i.domain, - id: i.id, - } -} - -func (s *Server) hasher(workerID int) { - for { - select { - case i := <-s.hashingQueue: - start := time.Now() - - hash := hashImage(i) - if hash.domain ==""{ - continue - } - - s.mappingQueue <- hash - t := time.Now() - elapsed := t.Sub(start) - // fmt.Printf("%#064b\n", ahash.GetHash()) - // fmt.Printf("%#064b\n", dhash.GetHash()) - // fmt.Printf("%#064b\n", phash.GetHash()) - fmt.Printf("Hashing took %v: worker: %v\n", elapsed, workerID) - } + return Hash{ + Ahash: ahash, + Dhash: dhash, + Phash: phash, + Domain: i.Domain, + ID: i.ID, } } func SplitHash(hash uint64) [8]uint8 { return [8]uint8{ - uint8((hash & h_8) >> shift_8), - uint8((hash & h_7) >> shift_7), - uint8((hash & h_6) >> shift_6), - uint8((hash & h_5) >> shift_5), - uint8((hash & h_4) >> shift_4), - uint8((hash & h_3) >> shift_3), - uint8((hash & h_2) >> shift_2), - uint8((hash & h_1) >> shift_1), + uint8((hash & H_7) >> Shift_7), + uint8((hash & H_6) >> Shift_6), + uint8((hash & H_5) >> Shift_5), + uint8((hash & H_4) >> Shift_4), + uint8((hash & H_3) >> Shift_3), + uint8((hash & H_2) >> Shift_2), + uint8((hash & H_1) >> Shift_1), + uint8((hash & H_0) >> Shift_0), } } -// func (s *Server) CoverByID(id string) uint32 { -// v,ok :=s.IDToCover[id] -// return 0 -// } -func (s *Server) FindHashes() { -} - -func startServer(cover_path string) { - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - fmt.Println("Fuck") - mux := http.NewServeMux() - server := Server{ - token: make(chan *oidc.Tokens), - hashingQueue: make(chan im, 5000), - mappingQueue: make(chan hash, 5000000), - mux: mux, - httpServer: &http.Server{ - Addr: ":8080", - Handler: mux, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - }, - } - fmt.Println("init hashes") - server.initHashes() - // server.setupOauthHandlers() - fmt.Println("init handlers") - server.setupAppHandlers() - fmt.Println("init hashers") - go server.hasher(1) - go server.hasher(2) - go server.hasher(3) - go server.hasher(4) - go server.hasher(5) - go server.hasher(6) - go server.hasher(7) - go server.hasher(8) - go server.hasher(9) - go server.hasher(10) - - fmt.Println("init mapper") - go server.mapper() - - fmt.Println("Starting local hashing go routine") - go func() { - fmt.Println("Hashing covers at ", cover_path) - err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error { - select { - case s := <-sig: - server.httpServer.Shutdown(context.TODO()) - return fmt.Errorf("Signal: %v", s) - default: - } - if d.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - panic(err) - } - i, format, err := image.Decode(bufio.NewReader(file)) - if err != nil { - return nil // skip this image - } - file.Close() - // fmt.Println("Hashing", path) - server.hashingQueue <- im{i, format, "comicvine.gamespot.com", filepath.Base(filepath.Dir(path))} - return nil - }) - fmt.Println(err) - }() - fmt.Println("Listening on ", server.httpServer.Addr) - err := server.httpServer.ListenAndServe() - if err != nil { - fmt.Println(err) - } - f, er := os.Create("memprofile") - if er != nil { - fmt.Println("Error in creating file for writing memory profile to: ", er) - return - } - defer f.Close() - runtime.GC() - if e := pprof.WriteHeapProfile(f); e != nil { - fmt.Println("Error in writing memory profile: ", e) - return - } -} +type IDList map[Source][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"} diff --git a/privdrop.go b/privdrop.go deleted file mode 100644 index 9d606fe..0000000 --- a/privdrop.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "syscall" - "unsafe" -) - -type capHeader struct { - version uint32 - pid int -} - -type capData struct { - effective uint32 - permitted uint32 - inheritable uint32 -} - -type caps struct { - hdr capHeader - data [2]capData -} - -func getCaps() (caps, error) { - var c caps - - // Get capability version - if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(nil)), 0); errno != 0 { - return c, fmt.Errorf("SYS_CAPGET: %v", errno) - } - - // Get current capabilities - if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(&c.data[0])), 0); errno != 0 { - return c, fmt.Errorf("SYS_CAPGET: %v", errno) - } - - return c, nil -} - -// mustDropPrivileges executes the program in a child process, dropping root -// privileges, but retaining the CAP_SYS_TIME capability to change the system -// clock. -func mustDropPrivileges() { - if os.Getenv("PRIVILEGES_DROPPED") == "1" { - return - } - - // From include/uapi/linux/capability.h: - // Allow setting the real-time clock - const CAP_SYS_TIME = 25 - - caps, err := getCaps() - if err != nil { - log.Fatal(err) - } - - // Add CAP_SYS_TIME to the permitted and inheritable capability mask, - // otherwise we will not be able to add it to the ambient capability mask. - caps.data[0].permitted |= 1 << uint(CAP_SYS_TIME) - caps.data[0].inheritable |= 1 << uint(CAP_SYS_TIME) - - if _, _, errno := syscall.Syscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&caps.hdr)), uintptr(unsafe.Pointer(&caps.data[0])), 0); errno != 0 { - log.Fatalf("SYS_CAPSET: %v", errno) - } - - cmd := exec.Command(os.Args[0]) - cmd.Env = append(os.Environ(), "PRIVILEGES_DROPPED=1") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.SysProcAttr = &syscall.SysProcAttr{ - Credential: &syscall.Credential{ - Uid: 65534, - Gid: 65534, - }, - AmbientCaps: []uintptr{CAP_SYS_TIME}, - } - log.Fatal(cmd.Run()) -}