From 4a7e61e189957ecfe2b080440b18b34b604a9596 Mon Sep 17 00:00:00 2001 From: Timmy Welch Date: Sun, 4 Aug 2024 18:26:20 -0700 Subject: [PATCH] static check fixes --- cmd/comic-hasher/main.go | 233 +++++++++++++++------------------ cmd/hash/cover_extract/main.go | 12 +- cmd/hash/main.go | 10 +- main.go | 64 ++++----- 4 files changed, 151 insertions(+), 168 deletions(-) diff --git a/cmd/comic-hasher/main.go b/cmd/comic-hasher/main.go index 92e511c..c9802e6 100644 --- a/cmd/comic-hasher/main.go +++ b/cmd/comic-hasher/main.go @@ -43,12 +43,12 @@ type Server struct { mux *http.ServeMux BaseURL *url.URL // token chan<- *oidc.Tokens - Partial_ahash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes - Partial_dhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes - Partial_phash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes - Full_ahash map[uint64]ch.IDList // Maps ahash's to lists of ID's - Full_dhash map[uint64]ch.IDList // Maps dhash's to lists of ID's - Full_phash map[uint64]ch.IDList // Maps phash's to lists of ID's + PartialAhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + PartialDhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + PartialPhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes + FullAhash map[uint64]ch.IDList // Maps ahash's to lists of ID's + FullDhash map[uint64]ch.IDList // Maps dhash's to lists of ID's + FullPhash map[uint64]ch.IDList // Maps phash's to lists of ID's // IDToCover map[string]string // IDToCover is a map of domain:ID to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0 // covers []ch.Cover readerQueue chan string @@ -66,17 +66,17 @@ func main() { }() // mustDropPrivileges() - cover_path := flag.String("cover_path", "", "path to covers to add to hash database") + coverPath := flag.String("cover_path", "", "path to covers to add to hash database") flag.Parse() - if *cover_path == "" { + if *coverPath == "" { log.Fatal("You must supply a path") } - st, err := os.Stat(*cover_path) + st, err := os.Stat(*coverPath) if err != nil { panic(err) } fmt.Println(st) - startServer(*cover_path) + startServer(*coverPath) } func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) { @@ -129,21 +129,21 @@ func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, // } func (s *Server) setupAppHandlers() { - s.mux.HandleFunc("/add_cover", s.add_cover) - s.mux.HandleFunc("/get_cover", s.get_cover) - s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash) + s.mux.HandleFunc("/add_cover", s.addCover) + s.mux.HandleFunc("/get_cover", s.getCover) + s.mux.HandleFunc("/match_cover_hash", s.matchCoverHash) } -func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) { +func (s *Server) getCover(w http.ResponseWriter, r *http.Request) { user, authed := s.authenticated(w, r) if !authed || user == "" { http.Error(w, "Invalid Auth", http.StatusForbidden) return } var ( - values url.Values = r.URL.Query() - domain = strings.TrimSpace(values.Get("domain")) - ID = strings.TrimSpace(values.Get("id")) + values = r.URL.Query() + domain = strings.TrimSpace(values.Get("domain")) + ID = strings.TrimSpace(values.Get("id")) ) if ID == "" { log.Println("No ID Provided") @@ -167,17 +167,16 @@ func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) { } func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result { - var foundMatches []ch.Result - if matchedResults, ok := s.Full_ahash[ahash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.AHash}}) + if matchedResults, ok := s.FullAhash[ahash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.AHash}}) } - if matchedResults, ok := s.Full_dhash[dhash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.DHash}}) + if matchedResults, ok := s.FullDhash[dhash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.DHash}}) } - if matchedResults, ok := s.Full_phash[phash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.PHash}}) + if matchedResults, ok := s.FullPhash[phash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.PHash}}) } // If we have exact matches don't bother with other matches @@ -185,26 +184,26 @@ func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result { return foundMatches } - for i, partial_hash := range ch.SplitHash(ahash) { - for _, match := range ch.Atleast(8, ahash, s.Partial_ahash[i][partial_hash]) { - if matchedResults, ok := s.Full_ahash[match.Hash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.AHash}}) + for i, partialHash := range ch.SplitHash(ahash) { + for _, match := range ch.Atleast(8, ahash, s.PartialAhash[i][partialHash]) { + if matchedResults, ok := s.FullAhash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.AHash}}) } } } - for i, partial_hash := range ch.SplitHash(dhash) { - for _, match := range ch.Atleast(8, dhash, s.Partial_dhash[i][partial_hash]) { - if matchedResults, ok := s.Full_dhash[match.Hash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.DHash}}) + for i, partialHash := range ch.SplitHash(dhash) { + for _, match := range ch.Atleast(8, dhash, s.PartialDhash[i][partialHash]) { + if matchedResults, ok := s.FullDhash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.DHash}}) } } } - for i, partial_hash := range ch.SplitHash(phash) { - for _, match := range ch.Atleast(8, phash, s.Partial_phash[i][partial_hash]) { - if matchedResults, ok := s.Full_phash[match.Hash]; ok { - foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.PHash}}) + for i, partialHash := range ch.SplitHash(phash) { + for _, match := range ch.Atleast(8, phash, s.PartialPhash[i][partialHash]) { + if matchedResults, ok := s.FullPhash[match.Hash]; ok { + foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.PHash}}) } } } @@ -212,7 +211,7 @@ func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result { return foundMatches } -func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) { +func (s *Server) matchCoverHash(w http.ResponseWriter, r *http.Request) { user, authed := s.authenticated(w, r) if !authed || user == "" { http.Error(w, "Invalid Auth", http.StatusForbidden) @@ -257,7 +256,7 @@ func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}") } -func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) { +func (s *Server) addCover(w http.ResponseWriter, r *http.Request) { user, authed := s.authenticated(w, r) if !authed || user == "" { http.Error(w, "Invalid Auth", http.StatusForbidden) @@ -291,118 +290,104 @@ func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) { } func (s *Server) mapHashes(hash ch.Hash) { - _, ahash_ok := s.Full_ahash[hash.Ahash.GetHash()] - if !ahash_ok { - s.Full_ahash[hash.Ahash.GetHash()] = make(ch.IDList) + if _, ok := s.FullAhash[hash.Ahash.GetHash()]; !ok { + s.FullAhash[hash.Ahash.GetHash()] = make(ch.IDList) } - s.Full_ahash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.Full_ahash[hash.Ahash.GetHash()][hash.Domain], hash.ID) - _, dhash_ok := s.Full_dhash[hash.Dhash.GetHash()] - if !dhash_ok { - s.Full_dhash[hash.Dhash.GetHash()] = make(ch.IDList) - } - s.Full_dhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.Full_dhash[hash.Dhash.GetHash()][hash.Domain], hash.ID) - _, phash_ok := s.Full_phash[hash.Phash.GetHash()] - if !phash_ok { - s.Full_phash[hash.Phash.GetHash()] = make(ch.IDList) - } - s.Full_phash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.Full_phash[hash.Phash.GetHash()][hash.Domain], hash.ID) + s.FullAhash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.FullAhash[hash.Ahash.GetHash()][hash.Domain], hash.ID) - for i, partial_hash := range ch.SplitHash(hash.Ahash.GetHash()) { - s.Partial_ahash[i][partial_hash] = ch.Insert(s.Partial_ahash[i][partial_hash], hash.Ahash.GetHash()) + if _, ok := s.FullDhash[hash.Dhash.GetHash()]; !ok { + s.FullDhash[hash.Dhash.GetHash()] = make(ch.IDList) } - for i, partial_hash := range ch.SplitHash(hash.Dhash.GetHash()) { - s.Partial_dhash[i][partial_hash] = ch.Insert(s.Partial_dhash[i][partial_hash], hash.Dhash.GetHash()) + s.FullDhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.FullDhash[hash.Dhash.GetHash()][hash.Domain], hash.ID) + + if _, ok := s.FullPhash[hash.Phash.GetHash()]; !ok { + s.FullPhash[hash.Phash.GetHash()] = make(ch.IDList) } - for i, partial_hash := range ch.SplitHash(hash.Phash.GetHash()) { - s.Partial_phash[i][partial_hash] = ch.Insert(s.Partial_phash[i][partial_hash], hash.Phash.GetHash()) + s.FullPhash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.FullPhash[hash.Phash.GetHash()][hash.Domain], hash.ID) + + for i, partialHash := range ch.SplitHash(hash.Ahash.GetHash()) { + s.PartialAhash[i][partialHash] = ch.Insert(s.PartialAhash[i][partialHash], hash.Ahash.GetHash()) + } + for i, partialHash := range ch.SplitHash(hash.Dhash.GetHash()) { + s.PartialDhash[i][partialHash] = ch.Insert(s.PartialDhash[i][partialHash], hash.Dhash.GetHash()) + } + for i, partialHash := range ch.SplitHash(hash.Phash.GetHash()) { + s.PartialPhash[i][partialHash] = ch.Insert(s.PartialPhash[i][partialHash], hash.Phash.GetHash()) } } func (s *Server) initHashes() { - for i := range s.Partial_ahash { - s.Partial_ahash[i] = make(map[uint8][]uint64) + for i := range s.PartialAhash { + s.PartialAhash[i] = make(map[uint8][]uint64) } - for i := range s.Partial_dhash { - s.Partial_dhash[i] = make(map[uint8][]uint64) + for i := range s.PartialDhash { + s.PartialDhash[i] = make(map[uint8][]uint64) } - for i := range s.Partial_phash { - s.Partial_phash[i] = make(map[uint8][]uint64) + for i := range s.PartialPhash { + s.PartialPhash[i] = make(map[uint8][]uint64) } - s.Full_ahash = make(map[uint64]ch.IDList) - s.Full_dhash = make(map[uint64]ch.IDList) - s.Full_phash = make(map[uint64]ch.IDList) + s.FullAhash = make(map[uint64]ch.IDList) + s.FullDhash = make(map[uint64]ch.IDList) + s.FullPhash = make(map[uint64]ch.IDList) // s.IDToCover = make(map[string]string) } func (s *Server) mapper() { var total uint64 = 0 - for { - select { - case hash := <-s.mappingQueue: - if total%1000 == 0 { - mem := ch.MemStats() - if mem > 10*1024*1024*1024 { - fmt.Println("Forcing gc", mem, "G") - runtime.GC() - } + for hash := range s.mappingQueue { + if total%1000 == 0 { + mem := ch.MemStats() + if mem > 10*1024*1024*1024 { + fmt.Println("Forcing gc", mem, "G") + runtime.GC() } - total++ - - s.mapHashes(hash) } + total++ + + s.mapHashes(hash) } } func (s *Server) hasher(workerID int) { - for { - select { - case i := <-s.hashingQueue: - start := time.Now() + for image := range s.hashingQueue { + start := time.Now() - hash := ch.HashImage(i) - if hash.Domain == "" { - continue - } - - s.mappingQueue <- hash - - elapsed := time.Now().Sub(start) - // fmt.Printf("%#064b\n", ahash.GetHash()) - // fmt.Printf("%#064b\n", dhash.GetHash()) - // fmt.Printf("%#064b\n", phash.GetHash()) - log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, i.Path, hash.Ahash.GetHash(), hash.ID) + hash := ch.HashImage(image) + if hash.Domain == "" { + continue } + + s.mappingQueue <- hash + + elapsed := time.Since(start) + // fmt.Printf("%#064b\n", ahash.GetHash()) + // fmt.Printf("%#064b\n", dhash.GetHash()) + // fmt.Printf("%#064b\n", phash.GetHash()) + log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, image.Path, hash.Ahash.GetHash(), hash.ID) } } func (s *Server) reader(workerID int) { - for { - select { - case path := <-s.readerQueue: - file, err := os.Open(path) - if err != nil { - panic(err) - } - i, format, err := image.Decode(bufio.NewReader(file)) - if err != nil { - continue // skip this image - } - file.Close() - // fmt.Println("Hashing", path) - im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path} - s.hashingQueue <- im + for path := range s.readerQueue { + file, err := os.Open(path) + if err != nil { + panic(err) } + i, format, err := image.Decode(bufio.NewReader(file)) + if err != nil { + continue // skip this image + } + file.Close() + + im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path} + s.hashingQueue <- im } } -// func (s *Server) CoverByID(ID string) uint32 { -// v,ok :=s.IDToCover[ID] -// return 0 -// } func (s *Server) FindHashes() { } -func startServer(cover_path string) { +func startServer(coverPath string) { if *cpuprofile != "" { f, err := os.Create(*cpuprofile) if err != nil { @@ -462,30 +447,28 @@ func startServer(cover_path string) { fmt.Println("Starting local hashing go routine") go func() { - fmt.Println("Hashing covers at ", cover_path) + fmt.Println("Hashing covers at ", coverPath) start := time.Now() - err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(coverPath, func(path string, d fs.DirEntry, err error) error { select { case s := <-sig: server.httpServer.Shutdown(context.TODO()) - return fmt.Errorf("Signal: %v", s) + return fmt.Errorf("signal: %v", s) default: } - if d.IsDir() { // Only hash thumbnails for now + if d.IsDir() { return nil } fmt.Println(len(server.readerQueue)) server.readerQueue <- path return nil }) - elapsed := time.Now().Sub(start) + elapsed := time.Since(start) fmt.Println("Err:", err, "local hashing took", elapsed) - select { - case s := <-sig: - server.httpServer.Shutdown(context.TODO()) - log.Printf("Signal: %v", s) - } + s := <-sig + err = server.httpServer.Shutdown(context.TODO()) + log.Printf("Signal: %v, error: %s", s, err) }() fmt.Println("Listening on ", server.httpServer.Addr) diff --git a/cmd/hash/cover_extract/main.go b/cmd/hash/cover_extract/main.go index 920f013..ad6f740 100644 --- a/cmd/hash/cover_extract/main.go +++ b/cmd/hash/cover_extract/main.go @@ -46,9 +46,11 @@ func main() { panic(err) } c.SortStrings(fileList) - var image []byte - var issue_id string - var files = []string{"ComicInfo.xml", fileList[0]} + var ( + image []byte + issueID string + files = []string{"ComicInfo.xml", fileList[0]} + ) fmt.Printf("Extracting %s\n", fileList[0]) err = unrar.Extract(context.TODO(), file, files, func(ctx context.Context, f archiver.File) error { r, err := f.Open() @@ -62,7 +64,7 @@ func main() { } parts := strings.Split(strings.TrimRight(ci.Web, "/"), "/") ids := strings.Split(parts[len(parts)-1], "-") - issue_id = ids[1] + issueID = ids[1] } else { image, err = io.ReadAll(r) if err != nil { @@ -75,7 +77,7 @@ func main() { panic(err) } file.Close() - file, err = os.Create(*fileArchive + "." + issue_id + ".image") + file, err = os.Create(*fileArchive + "." + issueID + ".image") if err != nil { panic(err) } diff --git a/cmd/hash/main.go b/cmd/hash/main.go index ff452b4..a7f819f 100644 --- a/cmd/hash/main.go +++ b/cmd/hash/main.go @@ -23,10 +23,9 @@ func init() { // DisableBlockSmoothing: false, // DCTMethod: jpeg.DCTFloat, // })}, jpeg.DecodeConfig) - } -func save_image(im image.Image, name string) { +func saveImage(im image.Image, name string) { file, err := os.Create(name) if err != nil { log.Printf("Failed to open file %s: %s", "tmp.png", err) @@ -63,18 +62,17 @@ func debugImage(im image.Image, width, height int) { gray := goimagehash.ToGray(im, nil) resized := goimagehash.Resize(gray, width, height, nil) - save_image(im, "go.rgb.png") + saveImage(im, "go.rgb.png") log.Println("rgb") log.Println(fmtImage(im)) - save_image(gray, "go.gray.png") + saveImage(gray, "go.gray.png") log.Println("gray") log.Println(fmtImage(gray)) - save_image(resized, "go.resized.png") + saveImage(resized, "go.resized.png") log.Println("resized") log.Println(fmtImage(resized)) - } func main() { diff --git a/main.go b/main.go index 30e4ab3..a34eaaf 100644 --- a/main.go +++ b/main.go @@ -13,25 +13,25 @@ import ( ) const ( - H_0 uint64 = 0b11111111 << (8 * iota) - H_1 - H_2 - H_3 - H_4 - H_5 - H_6 - H_7 + H0 uint64 = 0b11111111 << (8 * iota) + H1 + H2 + H3 + H4 + H5 + H6 + H7 ) const ( - Shift_0 = (8 * iota) - Shift_1 - Shift_2 - Shift_3 - Shift_4 - Shift_5 - Shift_6 - Shift_7 + Shift0 = (8 * iota) + Shift1 + Shift2 + Shift3 + Shift4 + Shift5 + Shift6 + Shift7 ) type Source string @@ -67,20 +67,20 @@ type ImageHash struct { Kind goimagehash.Kind } -func Atleast(maxDistance int, search_hash uint64, hashes []uint64) []Match { - matching_hashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them - for _, stored_hash := range hashes { - distance := bits.OnesCount64(search_hash ^ stored_hash) +func Atleast(maxDistance int, searchHash uint64, hashes []uint64) []Match { + matchingHashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them + for _, storedHash := range hashes { + distance := bits.OnesCount64(searchHash ^ storedHash) if distance <= maxDistance { - matching_hashes = append(matching_hashes, Match{distance, stored_hash}) + matchingHashes = append(matchingHashes, Match{distance, storedHash}) } } - return matching_hashes + return matchingHashes } func Insert[S ~[]E, E cmp.Ordered](slice S, item E) S { - index, item_found := slices.BinarySearch(slice, item) - if item_found { + index, itemFound := slices.BinarySearch(slice, item) + if itemFound { return slice } return slices.Insert(slice, index, item) @@ -133,14 +133,14 @@ func HashImage(i Im) Hash { func SplitHash(hash uint64) [8]uint8 { return [8]uint8{ - uint8((hash & H_7) >> Shift_7), - uint8((hash & H_6) >> Shift_6), - uint8((hash & H_5) >> Shift_5), - uint8((hash & H_4) >> Shift_4), - uint8((hash & H_3) >> Shift_3), - uint8((hash & H_2) >> Shift_2), - uint8((hash & H_1) >> Shift_1), - uint8((hash & H_0) >> Shift_0), + uint8((hash & H7) >> Shift7), + uint8((hash & H6) >> Shift6), + uint8((hash & H5) >> Shift5), + uint8((hash & H4) >> Shift4), + uint8((hash & H3) >> Shift3), + uint8((hash & H2) >> Shift2), + uint8((hash & H1) >> Shift1), + uint8((hash & H0) >> Shift0), } }