lordwelch/comic-hasher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more stuff

Browse Source
This commit is contained in:
Timmy Welch 2024-08-04 18:12:00 -07:00
parent c61ade9961
commit e2082465c6
7 changed files with 650 additions and 741 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

507
cmd/comic-hasher/main.go Normal file
View File

@ -0,0 +1,507 @@
package main
import (
"bufio"
"context"
"encoding/json"
"flag"
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"io/fs"
"log"
"net/http"
_ "net/http/pprof"
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"time"
"github.com/disintegration/imaging"
_ "golang.org/x/image/tiff"
_ "golang.org/x/image/vp8"
_ "golang.org/x/image/vp8l"
_ "golang.org/x/image/webp"
ch "gitea.narnian.us/lordwelch/comic-hasher"
"gitea.narnian.us/lordwelch/goimagehash"
// "github.com/google/uuid"
// "github.com/zitadel/oidc/pkg/client/rp"
// httphelper "github.com/zitadel/oidc/pkg/http"
// "github.com/zitadel/oidc/pkg/oidc"
)
type Server struct {
httpServer *http.Server
mux *http.ServeMux
BaseURL *url.URL
// token chan<- *oidc.Tokens
Partial_ahash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Partial_dhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Partial_phash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Full_ahash map[uint64]ch.IDList // Maps ahash's to lists of ID's
Full_dhash map[uint64]ch.IDList // Maps dhash's to lists of ID's
Full_phash map[uint64]ch.IDList // Maps phash's to lists of ID's
// IDToCover map[string]string // IDToCover is a map of domain:ID to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0
// covers []ch.Cover
readerQueue chan string
hashingQueue chan ch.Im
mappingQueue chan ch.Hash
// hashes are a uint64 split into 8 pieces or a unint64 for quick lookup, the value is an index to covers
}
// var key = []byte(uuid.New().String())[:16]
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
func main() {
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
// mustDropPrivileges()
cover_path := flag.String("cover_path", "", "path to covers to add to hash database")
flag.Parse()
if *cover_path == "" {
log.Fatal("You must supply a path")
}
st, err := os.Stat(*cover_path)
if err != nil {
panic(err)
}
fmt.Println(st)
startServer(*cover_path)
}
func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) {
return strings.TrimSpace("lordwelch"), true
}
// func (s *Server) setupOauthHandlers() error {
// redirectURI := *s.BaseURL
// redirectURI.Path = "/oauth/callback"
// successURI := *s.BaseURL
// successURI.Path = "/success"
// failURI := *s.BaseURL
// failURI.RawQuery = url.Values{"auth": []string{"fail"}}.Encode()
// cookieHandler := httphelper.NewCookieHandler(key, key, httphelper.WithUnsecure())
// options := []rp.Option{
// rp.WithCookieHandler(cookieHandler),
// rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)),
// }
// provider, err := rp.NewRelyingPartyOIDC(os.Getenv("COMICHASHER_PROVIDER_URL"), os.Getenv("COMICHASHER_CLIENT_ID"), os.Getenv("COMICHASHER_CLIENT_SECRET"), redirectURI.String(), strings.Split(os.Getenv("COMICHASHER_SCOPES"), ","), options...)
// if err != nil {
// return fmt.Errorf("error creating provider: %w", err)
// }
// // generate some state (representing the state of the user in your application,
// // e.g. the page where he was before sending him to login
// state := func() string {
// return uuid.New().String()
// }
// // register the AuthURLHandler at your preferred path
// // the AuthURLHandler creates the auth request and redirects the user to the auth server
// // including state handling with secure cookie and the possibility to use PKCE
// s.mux.Handle("/login", rp.AuthURLHandler(state, provider))
// // for demonstration purposes the returned userinfo response is written as JSON object onto response
// marshalUserinfo := func(w http.ResponseWriter, r *http.Request, tokens *oidc.Tokens, state string, rp rp.RelyingParty) {
// s.token <- tokens
// w.Header().Add("location", successURI.String())
// w.WriteHeader(301)
// }
// // register the CodeExchangeHandler at the callbackPath
// // the CodeExchangeHandler handles the auth response, creates the token request and calls the callback function
// // with the returned tokens from the token endpoint
// s.mux.Handle(redirectURI.Path, rp.CodeExchangeHandler(marshalUserinfo, provider))
// return nil
// }
func (s *Server) setupAppHandlers() {
s.mux.HandleFunc("/add_cover", s.add_cover)
s.mux.HandleFunc("/get_cover", s.get_cover)
s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash)
}
func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values url.Values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
ID = strings.TrimSpace(values.Get("id"))
)
if ID == "" {
log.Println("No ID Provided")
http.Error(w, "No ID Provided", http.StatusBadRequest)
return
}
if domain == "" {
log.Println("No domain Provided")
http.Error(w, "No domain Provided", http.StatusBadRequest)
return
}
// if index, ok := s.IDToCover[domain+":"+ID]; ok {
// covers, err := json.Marshal(s.covers[index])
// if err == nil {
// w.Header().Add("Content-Type", "application/json")
// w.Write(covers)
// return
// }
// }
fmt.Fprintln(w, "Not implemented")
}
func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result {
var foundMatches []ch.Result
if matchedResults, ok := s.Full_ahash[ahash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.AHash}})
}
if matchedResults, ok := s.Full_dhash[dhash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.DHash}})
}
if matchedResults, ok := s.Full_phash[phash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.PHash}})
}
// If we have exact matches don't bother with other matches
if len(foundMatches) > 0 {
return foundMatches
}
for i, partial_hash := range ch.SplitHash(ahash) {
for _, match := range ch.Atleast(8, ahash, s.Partial_ahash[i][partial_hash]) {
if matchedResults, ok := s.Full_ahash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.AHash}})
}
}
}
for i, partial_hash := range ch.SplitHash(dhash) {
for _, match := range ch.Atleast(8, dhash, s.Partial_dhash[i][partial_hash]) {
if matchedResults, ok := s.Full_dhash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.DHash}})
}
}
}
for i, partial_hash := range ch.SplitHash(phash) {
for _, match := range ch.Atleast(8, phash, s.Partial_phash[i][partial_hash]) {
if matchedResults, ok := s.Full_phash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.PHash}})
}
}
}
return foundMatches
}
func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values = r.URL.Query()
ahashStr = strings.TrimSpace(values.Get("ahash"))
dhashStr = strings.TrimSpace(values.Get("dhash"))
phashStr = strings.TrimSpace(values.Get("phash"))
ahash uint64
dhash uint64
phash uint64
err error
)
if ahash, err = strconv.ParseUint(ahashStr, 16, 64); err != nil && ahashStr != "" {
log.Printf("could not parse ahash: %s", ahashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
if dhash, err = strconv.ParseUint(dhashStr, 16, 64); err != nil && dhashStr != "" {
log.Printf("could not parse dhash: %s", dhashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
if phash, err = strconv.ParseUint(phashStr, 16, 64); err != nil && phashStr != "" {
log.Printf("could not parse phash: %s", phashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
matches := s.getMatches(ahash, dhash, phash)
if len(matches) > 0 {
covers, err := json.Marshal(matches)
log.Println(err)
w.Header().Add("Content-Type", "application/json")
w.Write(covers)
w.Write([]byte{'\n'})
return
}
w.Header().Add("Content-Type", "application/json")
fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}")
}
func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
ID = strings.TrimSpace(values.Get("id"))
)
if ID == "" {
log.Println("No ID Provided")
http.Error(w, "No ID Provided", http.StatusBadRequest)
return
}
if domain == "" {
log.Println("No domain Provided")
http.Error(w, "No domain Provided", http.StatusBadRequest)
return
}
i, format, err := image.Decode(r.Body)
if err != nil {
msg := fmt.Sprintf("Failed to decode Image: %s", err)
log.Println(msg)
http.Error(w, msg, http.StatusBadRequest)
return
}
log.Printf("Decoded %s image from %s", format, user)
s.hashingQueue <- ch.Im{Im: i, Format: format, Domain: ch.Source(domain), ID: ID, Path: ""}
fmt.Fprintln(w, "Success")
}
func (s *Server) mapHashes(hash ch.Hash) {
_, ahash_ok := s.Full_ahash[hash.Ahash.GetHash()]
if !ahash_ok {
s.Full_ahash[hash.Ahash.GetHash()] = make(ch.IDList)
}
s.Full_ahash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.Full_ahash[hash.Ahash.GetHash()][hash.Domain], hash.ID)
_, dhash_ok := s.Full_dhash[hash.Dhash.GetHash()]
if !dhash_ok {
s.Full_dhash[hash.Dhash.GetHash()] = make(ch.IDList)
}
s.Full_dhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.Full_dhash[hash.Dhash.GetHash()][hash.Domain], hash.ID)
_, phash_ok := s.Full_phash[hash.Phash.GetHash()]
if !phash_ok {
s.Full_phash[hash.Phash.GetHash()] = make(ch.IDList)
}
s.Full_phash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.Full_phash[hash.Phash.GetHash()][hash.Domain], hash.ID)
for i, partial_hash := range ch.SplitHash(hash.Ahash.GetHash()) {
s.Partial_ahash[i][partial_hash] = ch.Insert(s.Partial_ahash[i][partial_hash], hash.Ahash.GetHash())
}
for i, partial_hash := range ch.SplitHash(hash.Dhash.GetHash()) {
s.Partial_dhash[i][partial_hash] = ch.Insert(s.Partial_dhash[i][partial_hash], hash.Dhash.GetHash())
}
for i, partial_hash := range ch.SplitHash(hash.Phash.GetHash()) {
s.Partial_phash[i][partial_hash] = ch.Insert(s.Partial_phash[i][partial_hash], hash.Phash.GetHash())
}
}
func (s *Server) initHashes() {
for i := range s.Partial_ahash {
s.Partial_ahash[i] = make(map[uint8][]uint64)
}
for i := range s.Partial_dhash {
s.Partial_dhash[i] = make(map[uint8][]uint64)
}
for i := range s.Partial_phash {
s.Partial_phash[i] = make(map[uint8][]uint64)
}
s.Full_ahash = make(map[uint64]ch.IDList)
s.Full_dhash = make(map[uint64]ch.IDList)
s.Full_phash = make(map[uint64]ch.IDList)
// s.IDToCover = make(map[string]string)
}
func (s *Server) mapper() {
var total uint64 = 0
for {
select {
case hash := <-s.mappingQueue:
if total%1000 == 0 {
mem := ch.MemStats()
if mem > 10*1024*1024*1024 {
fmt.Println("Forcing gc", mem, "G")
runtime.GC()
}
}
total++
s.mapHashes(hash)
}
}
}
func (s *Server) hasher(workerID int) {
for {
select {
case i := <-s.hashingQueue:
start := time.Now()
hash := ch.HashImage(i)
if hash.Domain == "" {
continue
}
s.mappingQueue <- hash
elapsed := time.Now().Sub(start)
// fmt.Printf("%#064b\n", ahash.GetHash())
// fmt.Printf("%#064b\n", dhash.GetHash())
// fmt.Printf("%#064b\n", phash.GetHash())
log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, i.Path, hash.Ahash.GetHash(), hash.ID)
}
}
}
func (s *Server) reader(workerID int) {
for {
select {
case path := <-s.readerQueue:
file, err := os.Open(path)
if err != nil {
panic(err)
}
i, format, err := image.Decode(bufio.NewReader(file))
if err != nil {
continue // skip this image
}
file.Close()
// fmt.Println("Hashing", path)
im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path}
s.hashingQueue <- im
}
}
}
// func (s *Server) CoverByID(ID string) uint32 {
// v,ok :=s.IDToCover[ID]
// return 0
// }
func (s *Server) FindHashes() {
}
func startServer(cover_path string) {
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
mux := http.NewServeMux()
server := Server{
// token: make(chan *oidc.Tokens),
readerQueue: make(chan string, 1120130), // Number gotten from checking queue size
hashingQueue: make(chan ch.Im),
mappingQueue: make(chan ch.Hash),
mux: mux,
httpServer: &http.Server{
Addr: ":8080",
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
},
}
imaging.SetMaxProcs(1)
fmt.Println("init hashes")
server.initHashes()
// server.setupOauthHandlers()
fmt.Println("init handlers")
server.setupAppHandlers()
fmt.Println("init hashers")
go server.reader(1)
go server.reader(2)
go server.reader(3)
go server.reader(4)
go server.reader(5)
go server.reader(6)
go server.reader(7)
go server.reader(8)
go server.reader(9)
go server.reader(10)
go server.hasher(1)
go server.hasher(2)
go server.hasher(3)
go server.hasher(4)
go server.hasher(5)
go server.hasher(6)
go server.hasher(7)
go server.hasher(8)
go server.hasher(9)
go server.hasher(10)
fmt.Println("init mapper")
go server.mapper()
fmt.Println("Starting local hashing go routine")
go func() {
fmt.Println("Hashing covers at ", cover_path)
start := time.Now()
err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error {
select {
case s := <-sig:
server.httpServer.Shutdown(context.TODO())
return fmt.Errorf("Signal: %v", s)
default:
}
if d.IsDir() { // Only hash thumbnails for now
return nil
}
fmt.Println(len(server.readerQueue))
server.readerQueue <- path
return nil
})
elapsed := time.Now().Sub(start)
fmt.Println("Err:", err, "local hashing took", elapsed)
select {
case s := <-sig:
server.httpServer.Shutdown(context.TODO())
log.Printf("Signal: %v", s)
}
}()
fmt.Println("Listening on ", server.httpServer.Addr)
err := server.httpServer.ListenAndServe()
if err != nil {
fmt.Println(err)
}
f, er := os.Create("memprofile")
if er != nil {
fmt.Println("Error in creating file for writing memory profile to: ", er)
return
}
defer f.Close()
runtime.GC()
if e := pprof.WriteHeapProfile(f); e != nil {
fmt.Println("Error in writing memory profile: ", e)
return
}
}

50
cmd/hash.py
View File

@ -1,10 +1,11 @@
from typing import Collection, Sequence
from PIL import Image
import argparse,pathlib,numpy,imagehash
import argparse,pathlib,numpy,imagehash,sys
ap = argparse.ArgumentParser()
ap.add_argument("--file", type=pathlib.Path)
ap.add_argument("--debug", action='store_true')
opts = ap.parse_args()
opts.file = pathlib.Path(opts.file)
@ -18,39 +19,42 @@ resized = gray.copy().resize((hash_size, hash_size), Image.Resampling.LANCZOS)
def print_image(image: Image.Image) -> None:
for row in numpy.asarray(image):
print('[ ', end='')
print('[ ', end='', file=sys.stderr)
for i in row:
if isinstance(i, Collection):
print('{ ', end='')
print('{ ', end='', file=sys.stderr)
for idx, x in enumerate(i):
if idx == len(i)-1:
print(f'{int(x):03d} ', end='')
print(f'{int(x):03d} ', end='', file=sys.stderr)
else:
print(f'{int(x):03d}, ', end='')
print('}, ', end='')
print(f'{int(x):03d}, ', end='', file=sys.stderr)
print('}, ', end='', file=sys.stderr)
else:
print(f'{int(i):03d}, ', end='')
print(']')
print(f'{int(i):03d}, ', end='', file=sys.stderr)
print(']', file=sys.stderr)
def bin_str(hash):
return ''.join(str(b) for b in 1 * hash.hash.flatten())
# print("rgb")
# print_image(image)
# print()
image.save("py.rgb.png")
if opts.debug:
image.save("py.rgb.png")
print("rgb", file=sys.stderr)
print_image(image)
print(file=sys.stderr)
# print("gray")
# print_image(gray)
gray.save("py.gray.png")
# print()
if opts.debug:
gray.save("py.gray.png")
print("gray", file=sys.stderr)
print_image(gray)
print(file=sys.stderr)
# print("resized")
# print_image(resized)
resized.save("py.resized.png")
# print()
if opts.debug:
resized.save("py.resized.png")
print("resized", file=sys.stderr)
print_image(resized)
print(file=sys.stderr)
print(str(imagehash.average_hash(image)))
print(str(imagehash.dhash(image)))
print(str(imagehash.phash(image)))
print('ahash: ', bin_str(imagehash.average_hash(image)))
print('dhash: ', bin_str(imagehash.dhash(image)))
print('phash: ', bin_str(imagehash.phash(image)))

93
cmd/hash/main.go
View File

@ -1,28 +1,20 @@
package main
import (
"bufio"
"flag"
"fmt"
"image"
"image/draw"
_ "image/gif"
_ "image/jpeg"
// "github.com/pixiv/go-libjpeg/jpeg"
"image/png"
"log"
"os"
"strings"
"gitea.narnian.us/lordwelch/goimagehash"
"gitea.narnian.us/lordwelch/goimagehash/transforms"
"github.com/anthonynsimon/bild/transform"
ih "gitea.narnian.us/lordwelch/image-hasher"
_ "github.com/gen2brain/avif"
_ "github.com/spakin/netpbm"
_ "golang.org/x/image/bmp"
_ "golang.org/x/image/tiff"
_ "golang.org/x/image/webp"
ch "gitea.narnian.us/lordwelch/comic-hasher"
"gitea.narnian.us/lordwelch/goimagehash"
)
func init() {
@ -34,27 +26,6 @@ func init() {
}
func ToGray(img image.Image, pix []uint8) *image.Gray {
c := img.Bounds().Dx() * img.Bounds().Dy()
if cap(pix) < c {
pix = append([]byte(nil), make([]byte, c)...)
}
pix = pix[:c]
gray := &image.Gray{
Pix: transforms.Rgb2Gray(img, pix),
Stride: img.Bounds().Dx(),
Rect: img.Bounds(),
}
return gray
}
func resize(img image.Image, w, h int) *image.Gray {
resized := transform.Resize(img, w, h, transform.Lanczos)
r_gray := image.NewGray(image.Rect(0, 0, resized.Bounds().Dx(), resized.Bounds().Dy()))
draw.Draw(r_gray, resized.Bounds(), resized, resized.Bounds().Min, draw.Src)
return r_gray
}
func save_image(im image.Image, name string) {
file, err := os.Create(name)
if err != nil {
@ -89,22 +60,27 @@ func fmtImage(im image.Image) string {
}
func debugImage(im image.Image, width, height int) {
gray := ToGray(im, nil)
resized := resize(gray, width, height)
gray := goimagehash.ToGray(im, nil)
resized := goimagehash.Resize(gray, width, height, nil)
save_image(im, "go.rgb.png")
log.Println("rgb")
log.Println(fmtImage(im))
save_image(im, "go.rgb.png")
save_image(gray, "go.gray.png")
log.Println("gray")
log.Println(fmtImage(gray))
save_image(gray, "go.gray.png")
save_image(resized, "go.resized.png")
log.Println("resized")
log.Println(fmtImage(resized))
save_image(resized, "go.resized.png")
}
func main() {
log.SetFlags(0)
imPath := flag.String("file", "", "image file to hash")
debug := flag.Bool("debug", false, "Enable debug output")
flag.Parse()
if imPath == nil || *imPath == "" {
flag.Usage()
@ -117,47 +93,24 @@ func main() {
return
}
defer file.Close()
im, format, err := image.Decode(file)
im, format, err := image.Decode(bufio.NewReader(file))
if err != nil {
msg := fmt.Sprintf("Failed to decode Image: %s", err)
log.Println(msg)
return
}
debugim := im
if format == "webp" {
im = goimagehash.FancyUpscale(im.(*image.YCbCr))
debugim = goimagehash.FancyUpscale(im.(*image.YCbCr))
}
debugImage(im, 8, 8)
var (
ahash *goimagehash.ImageHash
dhash *goimagehash.ImageHash
phash *goimagehash.ImageHash
)
ahash, err = goimagehash.AverageHash(im)
if err != nil {
msg := fmt.Sprintf("Failed to ahash Image: %s", err)
log.Println(msg)
return
if *debug {
debugImage(debugim, 8, 8)
}
dhash, err = goimagehash.DifferenceHash(im)
if err != nil {
msg := fmt.Sprintf("Failed to ahash Image: %s", err)
log.Println(msg)
return
}
hash := ch.HashImage(ch.Im{Im: im, Format: format, Domain: ch.Source("comicvine.gamespot.com"), ID: "nothing"})
phash, err = goimagehash.PerceptionHash(im)
if err != nil {
msg := fmt.Sprintf("Failed to ahash Image: %s", err)
log.Println(msg)
return
}
fmt.Println("ahash: ", ahash.BinString())
fmt.Println("dhash: ", dhash.BinString())
fmt.Println("phash: ", phash.BinString())
fmt.Println("ahash: ", hash.Ahash.BinString())
fmt.Println("dhash: ", hash.Dhash.BinString())
fmt.Println("phash: ", hash.Phash.BinString())
}

25
go.mod
View File

@ -1,4 +1,4 @@
module gitea.narnian.us/lordwelch/image-hasher
module gitea.narnian.us/lordwelch/comic-hasher
go 1.22.1
@ -6,13 +6,9 @@ toolchain go1.22.2
require (
gitea.narnian.us/lordwelch/goimagehash v0.0.0-20240502010648-cb5a8237c420
github.com/anthonynsimon/bild v0.13.0
github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09
github.com/fmartingr/go-comicinfo/v2 v2.0.2
github.com/gen2brain/avif v0.3.1
github.com/google/uuid v1.3.0
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/spakin/netpbm v1.3.0
github.com/zitadel/oidc v1.13.4
golang.org/x/image v0.7.0
)
@ -23,7 +19,7 @@ require (
github.com/bodgit/windows v1.0.0 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/ebitengine/purego v0.7.1 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@ -31,26 +27,13 @@ require (
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/tetratelabs/wazero v1.7.1 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
golang.org/x/sys v0.19.0 // indirect
)
require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/gorilla/schema v1.2.0 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/text v0.14.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
)
require golang.org/x/text v0.14.0
replace golang.org/x/text v0.14.0 => /home/timmy/build/source/text/

86
go.sum
View File

@ -19,9 +19,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/anthonynsimon/bild v0.13.0 h1:mN3tMaNds1wBWi1BrJq0ipDBhpkooYfu7ZFSMhXt1C8=
github.com/anthonynsimon/bild v0.13.0/go.mod h1:tpzzp0aYkAsMi1zmfhimaDyX1xjn2OUc1AJZK/TF0AE=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/bodgit/plumbing v1.2.0 h1:gg4haxoKphLjml+tgnecR4yLBV5zo4HAZGCtAh3xCzM=
github.com/bodgit/plumbing v1.2.0/go.mod h1:b9TeRi7Hvc6Y05rjm8VML3+47n4XTZPtQ/5ghqic2n8=
github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY=
@ -35,25 +32,18 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09 h1:MJFqtdxTq94XqUgg7DcGCaOIXrDTJE/tPHK66Jshguc=
github.com/disintegration/imaging v1.6.3-0.20201218193011-d40f48ce0f09/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA=
github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fmartingr/go-comicinfo/v2 v2.0.2 h1:VppvrHr8C4+iktBTOd7vzTMNbVecZ7F/Ji1kPTOIGg4=
github.com/fmartingr/go-comicinfo/v2 v2.0.2/go.mod h1:LUu/VenzEJkJt2PN49Kfpe50IgZkVkvH0m9Fnld8Dh0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gen2brain/avif v0.3.1 h1:womS2LKvhS/dSR3zIKUxtJW+riGlY48akGWqc+YgHtE=
github.com/gen2brain/avif v0.3.1/go.mod h1:s9sI2zo2cF6EdyRVCtnIfwL/Qb3k0TkOIEsz6ovK1ms=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -71,9 +61,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -82,35 +69,20 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc=
github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jeremija/gosubmit v0.2.7 h1:At0OhGCFGPXyjPYAsCchoBUhE099pcBXmsb4iZqROIc=
github.com/jeremija/gosubmit v0.2.7/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@ -123,68 +95,40 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spakin/netpbm v1.3.0 h1:eDX7VvrkN5sHXW0luZXRA4AKDlLmu0E5sNxJ7VSTwxc=
github.com/spakin/netpbm v1.3.0/go.mod h1:Q+ep6vNv1G44qSWp0wt3Y9o1m/QXjmaXZIFC0PMVpq0=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zitadel/logging v0.3.4 h1:9hZsTjMMTE3X2LUi0xcF9Q9EdLo+FAezeu52ireBbHM=
github.com/zitadel/logging v0.3.4/go.mod h1:aPpLQhE+v6ocNK0TWrBrd363hZ95KcI17Q1ixAQwZF0=
github.com/zitadel/oidc v1.13.4 h1:+k2GKqP9Ld9S2MSFlj+KaNsoZ3J9oy+Ezw51EzSFuC8=
github.com/zitadel/oidc v1.13.4/go.mod h1:3h2DhUcP02YV6q/CA/BG4yla0o6rXjK+DkJGK/dwJfw=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU=
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -196,8 +140,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -215,6 +159,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -232,27 +177,24 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -266,12 +208,12 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@ -309,11 +251,13 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@ -327,8 +271,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -349,15 +291,9 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

548
main.go
View File

@ -1,380 +1,100 @@
package main
package ch
import (
"bufio"
"context"
"encoding/json"
"flag"
"cmp"
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"io/fs"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"math/bits"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"time"
_ "golang.org/x/image/tiff"
_ "golang.org/x/image/vp8"
_ "golang.org/x/image/vp8l"
_ "golang.org/x/image/webp"
"slices"
"gitea.narnian.us/lordwelch/goimagehash"
"github.com/google/uuid"
"github.com/zitadel/oidc/pkg/client/rp"
httphelper "github.com/zitadel/oidc/pkg/http"
"github.com/zitadel/oidc/pkg/oidc"
)
const (
h_1 uint64 = 0b11111111 << (8 * iota)
h_2
h_3
h_4
h_5
h_6
h_7
h_8
H_0 uint64 = 0b11111111 << (8 * iota)
H_1
H_2
H_3
H_4
H_5
H_6
H_7
)
const (
shift_1 = (8 * iota)
shift_2
shift_3
shift_4
shift_5
shift_6
shift_7
shift_8
Shift_0 = (8 * iota)
Shift_1
Shift_2
Shift_3
Shift_4
Shift_5
Shift_6
Shift_7
)
type Cover map[string][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"}
type Source string
// type Cover struct {
// AHash uint64
// DHash uint64
// PHash uint64
// IDs map[string][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"}
// }
type Server struct {
httpServer *http.Server
mux *http.ServeMux
BaseURL *url.URL
token chan<- *oidc.Tokens
ahash [8]map[uint8]string
dhash [8]map[uint8]string
phash [8]map[uint8]string
fAhash map[uint64]string
fDhash map[uint64]string
fPhash map[uint64]string
IDToCover map[string]string // IDToCover is a map of domain:id to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0
covers []Cover
hashingQueue chan im
mappingQueue chan hash
// hashes are a uint64 split into 8 pieces or a unint64 for quick lookup, the value is an index to covers
type Match struct {
Distance int
Hash uint64
}
type im struct {
im image.Image
format string
domain string
id string
type Result struct {
IDs IDList
Distance int
Hash ImageHash
}
type hash struct {
ahash *goimagehash.ImageHash
dhash *goimagehash.ImageHash
phash *goimagehash.ImageHash
domain string
id string
type Im struct {
Im image.Image
Format string
Domain Source
ID, Path string
}
var key = []byte(uuid.New().String())[:16]
func main() {
// mustDropPrivileges()
cover_path := flag.String("cover_path", "", "path to covers to add to hash database")
flag.Parse()
if *cover_path == "" {
log.Fatal("You must supply a path")
}
st, err := os.Stat(*cover_path)
if err != nil {
panic(err)
}
fmt.Println(st)
startServer(*cover_path)
type Hash struct {
Ahash *goimagehash.ImageHash
Dhash *goimagehash.ImageHash
Phash *goimagehash.ImageHash
Domain Source
ID string
}
func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) {
return strings.TrimSpace("lordwelch"), true
type ImageHash struct {
Hash uint64
Kind goimagehash.Kind
}
func (s *Server) setupOauthHandlers() error {
redirectURI := *s.BaseURL
redirectURI.Path = "/oauth/callback"
successURI := *s.BaseURL
successURI.Path = "/success"
failURI := *s.BaseURL
failURI.RawQuery = url.Values{"auth": []string{"fail"}}.Encode()
cookieHandler := httphelper.NewCookieHandler(key, key, httphelper.WithUnsecure())
options := []rp.Option{
rp.WithCookieHandler(cookieHandler),
rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)),
}
provider, err := rp.NewRelyingPartyOIDC(os.Getenv("COMICHASHER_PROVIDER_URL"), os.Getenv("COMICHASHER_CLIENT_ID"), os.Getenv("COMICHASHER_CLIENT_SECRET"), redirectURI.String(), strings.Split(os.Getenv("COMICHASHER_SCOPES"), ","), options...)
if err != nil {
return fmt.Errorf("error creating provider: %w", err)
}
// generate some state (representing the state of the user in your application,
// e.g. the page where he was before sending him to login
state := func() string {
return uuid.New().String()
}
// register the AuthURLHandler at your preferred path
// the AuthURLHandler creates the auth request and redirects the user to the auth server
// including state handling with secure cookie and the possibility to use PKCE
s.mux.Handle("/login", rp.AuthURLHandler(state, provider))
// for demonstration purposes the returned userinfo response is written as JSON object onto response
marshalUserinfo := func(w http.ResponseWriter, r *http.Request, tokens *oidc.Tokens, state string, rp rp.RelyingParty) {
s.token <- tokens
w.Header().Add("location", successURI.String())
w.WriteHeader(301)
}
// register the CodeExchangeHandler at the callbackPath
// the CodeExchangeHandler handles the auth response, creates the token request and calls the callback function
// with the returned tokens from the token endpoint
s.mux.Handle(redirectURI.Path, rp.CodeExchangeHandler(marshalUserinfo, provider))
return nil
}
func (s *Server) setupAppHandlers() {
s.mux.HandleFunc("/add_cover", s.add_cover)
s.mux.HandleFunc("/get_cover", s.get_cover)
s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash)
}
func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
id = strings.TrimSpace(values.Get("id"))
)
if id == "" {
log.Println("No ID Provided")
http.Error(w, "No ID Provided", http.StatusBadRequest)
return
}
if domain == "" {
log.Println("No domain Provided")
http.Error(w, "No domain Provided", http.StatusBadRequest)
return
}
// if index, ok := s.IDToCover[domain+":"+id]; ok {
// covers, err := json.Marshal(s.covers[index])
// if err == nil {
// w.Header().Add("Content-Type", "application/json")
// w.Write(covers)
// return
// }
// }
fmt.Fprintln(w, "Not implemented")
}
func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values = r.URL.Query()
ahashStr = strings.TrimSpace(values.Get("ahash"))
dhashStr = strings.TrimSpace(values.Get("dhash"))
phashStr = strings.TrimSpace(values.Get("phash"))
ahash uint64
dhash uint64
phash uint64
err error
)
if ahash, err = strconv.ParseUint(ahashStr, 16, 64); err != nil && ahashStr != "" {
log.Printf("could not parse ahash: %s", ahashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
if dhash, err = strconv.ParseUint(dhashStr, 16, 64); err != nil && dhashStr != "" {
log.Printf("could not parse dhash: %s", dhashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
if phash, err = strconv.ParseUint(phashStr, 16, 64); err != nil && phashStr != "" {
log.Printf("could not parse phash: %s", phashStr)
http.Error(w, "parse fail", http.StatusBadRequest)
return
}
type res struct {
Domain string
Id string
}
if index, ok := s.fAhash[ahash]; ok {
covers, err := json.Marshal(res{
strings.Split(index, ":")[0],
strings.Split(index, ":")[1],
})
if err == nil {
w.Header().Add("Content-Type", "application/json")
w.Write(covers)
w.Write([]byte{'\n'})
return
} else {
fmt.Println(err)
func Atleast(maxDistance int, search_hash uint64, hashes []uint64) []Match {
matching_hashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them
for _, stored_hash := range hashes {
distance := bits.OnesCount64(search_hash ^ stored_hash)
if distance <= maxDistance {
matching_hashes = append(matching_hashes, Match{distance, stored_hash})
}
}
if index, ok := s.fDhash[dhash]; ok {
covers, err := json.Marshal(res{
strings.Split(index, ":")[0],
strings.Split(index, ":")[1],
})
if err == nil {
w.Header().Add("Content-Type", "application/json")
w.Write(covers)
w.Write([]byte{'\n'})
return
} else {
fmt.Println(err)
}
}
if index, ok := s.fPhash[phash]; ok {
covers, err := json.Marshal(res{
strings.Split(index, ":")[0],
strings.Split(index, ":")[1],
})
if err == nil {
w.Header().Add("Content-Type", "application/json")
w.Write(covers)
w.Write([]byte{'\n'})
return
} else {
fmt.Println(err)
}
}
w.Header().Add("Content-Type", "application/json")
fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}")
return matching_hashes
}
func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
func Insert[S ~[]E, E cmp.Ordered](slice S, item E) S {
index, item_found := slices.BinarySearch(slice, item)
if item_found {
return slice
}
var (
values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
id = strings.TrimSpace(values.Get("id"))
)
if id == "" {
log.Println("No ID Provided")
http.Error(w, "No ID Provided", http.StatusBadRequest)
return
}
if domain == "" {
log.Println("No domain Provided")
http.Error(w, "No domain Provided", http.StatusBadRequest)
return
}
i, format, err := image.Decode(r.Body)
if err != nil {
msg := fmt.Sprintf("Failed to decode Image: %s", err)
log.Println(msg)
http.Error(w, msg, http.StatusBadRequest)
return
}
log.Printf("Decoded %s image from %s", format, user)
s.hashingQueue <- im{i, format, domain, id}
fmt.Fprintln(w, "Success")
return slices.Insert(slice, index, item)
}
func (s *Server) mapHashes(id string, ahash, dhash, phash *goimagehash.ImageHash) {
s.fAhash[ahash.GetHash()] = id
s.fDhash[dhash.GetHash()] = id
s.fPhash[phash.GetHash()] = id
for i, partial_hash := range SplitHash(ahash.GetHash()) {
s.ahash[i][partial_hash] = id
}
for i, partial_hash := range SplitHash(dhash.GetHash()) {
s.dhash[i][partial_hash] = id
}
for i, partial_hash := range SplitHash(phash.GetHash()) {
s.phash[i][partial_hash] = id
}
}
func (s *Server) initHashes() {
for i := range s.ahash {
s.ahash[i] = make(map[uint8]string)
}
for i := range s.dhash {
s.dhash[i] = make(map[uint8]string)
}
for i := range s.phash {
s.phash[i] = make(map[uint8]string)
}
s.fAhash = make(map[uint64]string)
s.fDhash = make(map[uint64]string)
s.fPhash = make(map[uint64]string)
s.IDToCover = make(map[string]string)
}
func MemStats() uint64 {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return m.Alloc
}
func (s *Server) mapper() {
var total uint64 = 0
for {
select {
case hash := <-s.mappingQueue:
if total%1000 == 0 {
mem := MemStats()
if mem > 10*1024*1024*1024 {
fmt.Println("Forcing gc", mem, "G")
runtime.GC()
}
}
total++
s.mapHashes(hash.domain+":"+hash.id, hash.ahash, hash.dhash, hash.phash)
}
}
}
func hashImage(i im) hash {
if i.format == "webp" {
i.im = goimagehash.FancyUpscale(i.im.(*image.YCbCr))
func HashImage(i Im) Hash {
if i.Format == "webp" {
i.Im = goimagehash.FancyUpscale(i.Im.(*image.YCbCr))
}
var (
@ -384,156 +104,44 @@ func hashImage(i im) hash {
phash *goimagehash.ImageHash
)
ahash, err = goimagehash.AverageHash(i.im)
ahash, err = goimagehash.AverageHash(i.Im)
if err != nil {
msg := fmt.Sprintf("Failed to ahash Image: %s", err)
log.Println(msg)
return hash{}
return Hash{}
}
dhash, err = goimagehash.DifferenceHash(i.im)
dhash, err = goimagehash.DifferenceHash(i.Im)
if err != nil {
msg := fmt.Sprintf("Failed to dhash Image: %s", err)
log.Println(msg)
return hash{}
return Hash{}
}
phash, err = goimagehash.PerceptionHash(i.im)
phash, err = goimagehash.PerceptionHash(i.Im)
if err != nil {
msg := fmt.Sprintf("Failed to phash Image: %s", err)
log.Println(msg)
return hash{}
return Hash{}
}
return hash{
ahash: ahash,
dhash: dhash,
phash: phash,
domain: i.domain,
id: i.id,
}
}
func (s *Server) hasher(workerID int) {
for {
select {
case i := <-s.hashingQueue:
start := time.Now()
hash := hashImage(i)
if hash.domain ==""{
continue
}
s.mappingQueue <- hash
t := time.Now()
elapsed := t.Sub(start)
// fmt.Printf("%#064b\n", ahash.GetHash())
// fmt.Printf("%#064b\n", dhash.GetHash())
// fmt.Printf("%#064b\n", phash.GetHash())
fmt.Printf("Hashing took %v: worker: %v\n", elapsed, workerID)
}
return Hash{
Ahash: ahash,
Dhash: dhash,
Phash: phash,
Domain: i.Domain,
ID: i.ID,
}
}
func SplitHash(hash uint64) [8]uint8 {
return [8]uint8{
uint8((hash & h_8) >> shift_8),
uint8((hash & h_7) >> shift_7),
uint8((hash & h_6) >> shift_6),
uint8((hash & h_5) >> shift_5),
uint8((hash & h_4) >> shift_4),
uint8((hash & h_3) >> shift_3),
uint8((hash & h_2) >> shift_2),
uint8((hash & h_1) >> shift_1),
uint8((hash & H_7) >> Shift_7),
uint8((hash & H_6) >> Shift_6),
uint8((hash & H_5) >> Shift_5),
uint8((hash & H_4) >> Shift_4),
uint8((hash & H_3) >> Shift_3),
uint8((hash & H_2) >> Shift_2),
uint8((hash & H_1) >> Shift_1),
uint8((hash & H_0) >> Shift_0),
}
}
// func (s *Server) CoverByID(id string) uint32 {
// v,ok :=s.IDToCover[id]
// return 0
// }
func (s *Server) FindHashes() {
}
func startServer(cover_path string) {
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
fmt.Println("Fuck")
mux := http.NewServeMux()
server := Server{
token: make(chan *oidc.Tokens),
hashingQueue: make(chan im, 5000),
mappingQueue: make(chan hash, 5000000),
mux: mux,
httpServer: &http.Server{
Addr: ":8080",
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
},
}
fmt.Println("init hashes")
server.initHashes()
// server.setupOauthHandlers()
fmt.Println("init handlers")
server.setupAppHandlers()
fmt.Println("init hashers")
go server.hasher(1)
go server.hasher(2)
go server.hasher(3)
go server.hasher(4)
go server.hasher(5)
go server.hasher(6)
go server.hasher(7)
go server.hasher(8)
go server.hasher(9)
go server.hasher(10)
fmt.Println("init mapper")
go server.mapper()
fmt.Println("Starting local hashing go routine")
go func() {
fmt.Println("Hashing covers at ", cover_path)
err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error {
select {
case s := <-sig:
server.httpServer.Shutdown(context.TODO())
return fmt.Errorf("Signal: %v", s)
default:
}
if d.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
panic(err)
}
i, format, err := image.Decode(bufio.NewReader(file))
if err != nil {
return nil // skip this image
}
file.Close()
// fmt.Println("Hashing", path)
server.hashingQueue <- im{i, format, "comicvine.gamespot.com", filepath.Base(filepath.Dir(path))}
return nil
})
fmt.Println(err)
}()
fmt.Println("Listening on ", server.httpServer.Addr)
err := server.httpServer.ListenAndServe()
if err != nil {
fmt.Println(err)
}
f, er := os.Create("memprofile")
if er != nil {
fmt.Println("Error in creating file for writing memory profile to: ", er)
return
}
defer f.Close()
runtime.GC()
if e := pprof.WriteHeapProfile(f); e != nil {
fmt.Println("Error in writing memory profile: ", e)
return
}
}
type IDList map[Source][]string // IDs is a map of domain to ID eg IDs['comicvine.gamespot.com'] = []string{"1235"}

82
privdrop.go
View File

@ -1,82 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"syscall"
"unsafe"
)
type capHeader struct {
version uint32
pid int
}
type capData struct {
effective uint32
permitted uint32
inheritable uint32
}
type caps struct {
hdr capHeader
data [2]capData
}
func getCaps() (caps, error) {
var c caps
// Get capability version
if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(nil)), 0); errno != 0 {
return c, fmt.Errorf("SYS_CAPGET: %v", errno)
}
// Get current capabilities
if _, _, errno := syscall.Syscall(syscall.SYS_CAPGET, uintptr(unsafe.Pointer(&c.hdr)), uintptr(unsafe.Pointer(&c.data[0])), 0); errno != 0 {
return c, fmt.Errorf("SYS_CAPGET: %v", errno)
}
return c, nil
}
// mustDropPrivileges executes the program in a child process, dropping root
// privileges, but retaining the CAP_SYS_TIME capability to change the system
// clock.
func mustDropPrivileges() {
if os.Getenv("PRIVILEGES_DROPPED") == "1" {
return
}
// From include/uapi/linux/capability.h:
// Allow setting the real-time clock
const CAP_SYS_TIME = 25
caps, err := getCaps()
if err != nil {
log.Fatal(err)
}
// Add CAP_SYS_TIME to the permitted and inheritable capability mask,
// otherwise we will not be able to add it to the ambient capability mask.
caps.data[0].permitted |= 1 << uint(CAP_SYS_TIME)
caps.data[0].inheritable |= 1 << uint(CAP_SYS_TIME)
if _, _, errno := syscall.Syscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&caps.hdr)), uintptr(unsafe.Pointer(&caps.data[0])), 0); errno != 0 {
log.Fatalf("SYS_CAPSET: %v", errno)
}
cmd := exec.Command(os.Args[0])
cmd.Env = append(os.Environ(), "PRIVILEGES_DROPPED=1")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.SysProcAttr = &syscall.SysProcAttr{
Credential: &syscall.Credential{
Uid: 65534,
Gid: 65534,
},
AmbientCaps: []uintptr{CAP_SYS_TIME},
}
log.Fatal(cmd.Run())
}