static check fixes

This commit is contained in:
Timmy Welch 2024-08-04 18:26:20 -07:00
parent e2082465c6
commit 4a7e61e189
4 changed files with 151 additions and 168 deletions

View File

@ -43,12 +43,12 @@ type Server struct {
mux *http.ServeMux
BaseURL *url.URL
// token chan<- *oidc.Tokens
Partial_ahash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Partial_dhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Partial_phash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
Full_ahash map[uint64]ch.IDList // Maps ahash's to lists of ID's
Full_dhash map[uint64]ch.IDList // Maps dhash's to lists of ID's
Full_phash map[uint64]ch.IDList // Maps phash's to lists of ID's
PartialAhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
PartialDhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
PartialPhash [8]map[uint8][]uint64 // Maps partial hashes to their potential full hashes
FullAhash map[uint64]ch.IDList // Maps ahash's to lists of ID's
FullDhash map[uint64]ch.IDList // Maps dhash's to lists of ID's
FullPhash map[uint64]ch.IDList // Maps phash's to lists of ID's
// IDToCover map[string]string // IDToCover is a map of domain:ID to an index to covers eg IDToCover['comicvine.gamespot.com:12345'] = 0
// covers []ch.Cover
readerQueue chan string
@ -66,17 +66,17 @@ func main() {
}()
// mustDropPrivileges()
cover_path := flag.String("cover_path", "", "path to covers to add to hash database")
coverPath := flag.String("cover_path", "", "path to covers to add to hash database")
flag.Parse()
if *cover_path == "" {
if *coverPath == "" {
log.Fatal("You must supply a path")
}
st, err := os.Stat(*cover_path)
st, err := os.Stat(*coverPath)
if err != nil {
panic(err)
}
fmt.Println(st)
startServer(*cover_path)
startServer(*coverPath)
}
func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string, bool) {
@ -129,21 +129,21 @@ func (s *Server) authenticated(w http.ResponseWriter, r *http.Request) (string,
// }
func (s *Server) setupAppHandlers() {
s.mux.HandleFunc("/add_cover", s.add_cover)
s.mux.HandleFunc("/get_cover", s.get_cover)
s.mux.HandleFunc("/match_cover_hash", s.match_cover_hash)
s.mux.HandleFunc("/add_cover", s.addCover)
s.mux.HandleFunc("/get_cover", s.getCover)
s.mux.HandleFunc("/match_cover_hash", s.matchCoverHash)
}
func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) {
func (s *Server) getCover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
return
}
var (
values url.Values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
ID = strings.TrimSpace(values.Get("id"))
values = r.URL.Query()
domain = strings.TrimSpace(values.Get("domain"))
ID = strings.TrimSpace(values.Get("id"))
)
if ID == "" {
log.Println("No ID Provided")
@ -167,17 +167,16 @@ func (s *Server) get_cover(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result {
var foundMatches []ch.Result
if matchedResults, ok := s.Full_ahash[ahash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.AHash}})
if matchedResults, ok := s.FullAhash[ahash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.AHash}})
}
if matchedResults, ok := s.Full_dhash[dhash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.DHash}})
if matchedResults, ok := s.FullDhash[dhash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.DHash}})
}
if matchedResults, ok := s.Full_phash[phash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, 0, ch.ImageHash{ahash, goimagehash.PHash}})
if matchedResults, ok := s.FullPhash[phash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: 0, Hash: ch.ImageHash{Hash: ahash, Kind: goimagehash.PHash}})
}
// If we have exact matches don't bother with other matches
@ -185,26 +184,26 @@ func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result {
return foundMatches
}
for i, partial_hash := range ch.SplitHash(ahash) {
for _, match := range ch.Atleast(8, ahash, s.Partial_ahash[i][partial_hash]) {
if matchedResults, ok := s.Full_ahash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.AHash}})
for i, partialHash := range ch.SplitHash(ahash) {
for _, match := range ch.Atleast(8, ahash, s.PartialAhash[i][partialHash]) {
if matchedResults, ok := s.FullAhash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.AHash}})
}
}
}
for i, partial_hash := range ch.SplitHash(dhash) {
for _, match := range ch.Atleast(8, dhash, s.Partial_dhash[i][partial_hash]) {
if matchedResults, ok := s.Full_dhash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.DHash}})
for i, partialHash := range ch.SplitHash(dhash) {
for _, match := range ch.Atleast(8, dhash, s.PartialDhash[i][partialHash]) {
if matchedResults, ok := s.FullDhash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.DHash}})
}
}
}
for i, partial_hash := range ch.SplitHash(phash) {
for _, match := range ch.Atleast(8, phash, s.Partial_phash[i][partial_hash]) {
if matchedResults, ok := s.Full_phash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{matchedResults, match.Distance, ch.ImageHash{match.Hash, goimagehash.PHash}})
for i, partialHash := range ch.SplitHash(phash) {
for _, match := range ch.Atleast(8, phash, s.PartialPhash[i][partialHash]) {
if matchedResults, ok := s.FullPhash[match.Hash]; ok {
foundMatches = append(foundMatches, ch.Result{IDs: matchedResults, Distance: match.Distance, Hash: ch.ImageHash{Hash: match.Hash, Kind: goimagehash.PHash}})
}
}
}
@ -212,7 +211,7 @@ func (s *Server) getMatches(ahash, dhash, phash uint64) []ch.Result {
return foundMatches
}
func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) {
func (s *Server) matchCoverHash(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
@ -257,7 +256,7 @@ func (s *Server) match_cover_hash(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "{\"msg\":\"No hashes found\"}")
}
func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) {
func (s *Server) addCover(w http.ResponseWriter, r *http.Request) {
user, authed := s.authenticated(w, r)
if !authed || user == "" {
http.Error(w, "Invalid Auth", http.StatusForbidden)
@ -291,118 +290,104 @@ func (s *Server) add_cover(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) mapHashes(hash ch.Hash) {
_, ahash_ok := s.Full_ahash[hash.Ahash.GetHash()]
if !ahash_ok {
s.Full_ahash[hash.Ahash.GetHash()] = make(ch.IDList)
if _, ok := s.FullAhash[hash.Ahash.GetHash()]; !ok {
s.FullAhash[hash.Ahash.GetHash()] = make(ch.IDList)
}
s.Full_ahash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.Full_ahash[hash.Ahash.GetHash()][hash.Domain], hash.ID)
_, dhash_ok := s.Full_dhash[hash.Dhash.GetHash()]
if !dhash_ok {
s.Full_dhash[hash.Dhash.GetHash()] = make(ch.IDList)
}
s.Full_dhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.Full_dhash[hash.Dhash.GetHash()][hash.Domain], hash.ID)
_, phash_ok := s.Full_phash[hash.Phash.GetHash()]
if !phash_ok {
s.Full_phash[hash.Phash.GetHash()] = make(ch.IDList)
}
s.Full_phash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.Full_phash[hash.Phash.GetHash()][hash.Domain], hash.ID)
s.FullAhash[hash.Ahash.GetHash()][hash.Domain] = ch.Insert(s.FullAhash[hash.Ahash.GetHash()][hash.Domain], hash.ID)
for i, partial_hash := range ch.SplitHash(hash.Ahash.GetHash()) {
s.Partial_ahash[i][partial_hash] = ch.Insert(s.Partial_ahash[i][partial_hash], hash.Ahash.GetHash())
if _, ok := s.FullDhash[hash.Dhash.GetHash()]; !ok {
s.FullDhash[hash.Dhash.GetHash()] = make(ch.IDList)
}
for i, partial_hash := range ch.SplitHash(hash.Dhash.GetHash()) {
s.Partial_dhash[i][partial_hash] = ch.Insert(s.Partial_dhash[i][partial_hash], hash.Dhash.GetHash())
s.FullDhash[hash.Dhash.GetHash()][hash.Domain] = ch.Insert(s.FullDhash[hash.Dhash.GetHash()][hash.Domain], hash.ID)
if _, ok := s.FullPhash[hash.Phash.GetHash()]; !ok {
s.FullPhash[hash.Phash.GetHash()] = make(ch.IDList)
}
for i, partial_hash := range ch.SplitHash(hash.Phash.GetHash()) {
s.Partial_phash[i][partial_hash] = ch.Insert(s.Partial_phash[i][partial_hash], hash.Phash.GetHash())
s.FullPhash[hash.Phash.GetHash()][hash.Domain] = ch.Insert(s.FullPhash[hash.Phash.GetHash()][hash.Domain], hash.ID)
for i, partialHash := range ch.SplitHash(hash.Ahash.GetHash()) {
s.PartialAhash[i][partialHash] = ch.Insert(s.PartialAhash[i][partialHash], hash.Ahash.GetHash())
}
for i, partialHash := range ch.SplitHash(hash.Dhash.GetHash()) {
s.PartialDhash[i][partialHash] = ch.Insert(s.PartialDhash[i][partialHash], hash.Dhash.GetHash())
}
for i, partialHash := range ch.SplitHash(hash.Phash.GetHash()) {
s.PartialPhash[i][partialHash] = ch.Insert(s.PartialPhash[i][partialHash], hash.Phash.GetHash())
}
}
func (s *Server) initHashes() {
for i := range s.Partial_ahash {
s.Partial_ahash[i] = make(map[uint8][]uint64)
for i := range s.PartialAhash {
s.PartialAhash[i] = make(map[uint8][]uint64)
}
for i := range s.Partial_dhash {
s.Partial_dhash[i] = make(map[uint8][]uint64)
for i := range s.PartialDhash {
s.PartialDhash[i] = make(map[uint8][]uint64)
}
for i := range s.Partial_phash {
s.Partial_phash[i] = make(map[uint8][]uint64)
for i := range s.PartialPhash {
s.PartialPhash[i] = make(map[uint8][]uint64)
}
s.Full_ahash = make(map[uint64]ch.IDList)
s.Full_dhash = make(map[uint64]ch.IDList)
s.Full_phash = make(map[uint64]ch.IDList)
s.FullAhash = make(map[uint64]ch.IDList)
s.FullDhash = make(map[uint64]ch.IDList)
s.FullPhash = make(map[uint64]ch.IDList)
// s.IDToCover = make(map[string]string)
}
func (s *Server) mapper() {
var total uint64 = 0
for {
select {
case hash := <-s.mappingQueue:
if total%1000 == 0 {
mem := ch.MemStats()
if mem > 10*1024*1024*1024 {
fmt.Println("Forcing gc", mem, "G")
runtime.GC()
}
for hash := range s.mappingQueue {
if total%1000 == 0 {
mem := ch.MemStats()
if mem > 10*1024*1024*1024 {
fmt.Println("Forcing gc", mem, "G")
runtime.GC()
}
total++
s.mapHashes(hash)
}
total++
s.mapHashes(hash)
}
}
func (s *Server) hasher(workerID int) {
for {
select {
case i := <-s.hashingQueue:
start := time.Now()
for image := range s.hashingQueue {
start := time.Now()
hash := ch.HashImage(i)
if hash.Domain == "" {
continue
}
s.mappingQueue <- hash
elapsed := time.Now().Sub(start)
// fmt.Printf("%#064b\n", ahash.GetHash())
// fmt.Printf("%#064b\n", dhash.GetHash())
// fmt.Printf("%#064b\n", phash.GetHash())
log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, i.Path, hash.Ahash.GetHash(), hash.ID)
hash := ch.HashImage(image)
if hash.Domain == "" {
continue
}
s.mappingQueue <- hash
elapsed := time.Since(start)
// fmt.Printf("%#064b\n", ahash.GetHash())
// fmt.Printf("%#064b\n", dhash.GetHash())
// fmt.Printf("%#064b\n", phash.GetHash())
log.Printf("Hashing took %v: worker: %v. path: %s ahash: %064b id: %s\n", elapsed, workerID, image.Path, hash.Ahash.GetHash(), hash.ID)
}
}
func (s *Server) reader(workerID int) {
for {
select {
case path := <-s.readerQueue:
file, err := os.Open(path)
if err != nil {
panic(err)
}
i, format, err := image.Decode(bufio.NewReader(file))
if err != nil {
continue // skip this image
}
file.Close()
// fmt.Println("Hashing", path)
im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path}
s.hashingQueue <- im
for path := range s.readerQueue {
file, err := os.Open(path)
if err != nil {
panic(err)
}
i, format, err := image.Decode(bufio.NewReader(file))
if err != nil {
continue // skip this image
}
file.Close()
im := ch.Im{Im: i, Format: format, Domain: "comicvine.gamespot.com", ID: filepath.Base(filepath.Dir(path)), Path: path}
s.hashingQueue <- im
}
}
// func (s *Server) CoverByID(ID string) uint32 {
// v,ok :=s.IDToCover[ID]
// return 0
// }
func (s *Server) FindHashes() {
}
func startServer(cover_path string) {
func startServer(coverPath string) {
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
@ -462,30 +447,28 @@ func startServer(cover_path string) {
fmt.Println("Starting local hashing go routine")
go func() {
fmt.Println("Hashing covers at ", cover_path)
fmt.Println("Hashing covers at ", coverPath)
start := time.Now()
err := filepath.WalkDir(cover_path, func(path string, d fs.DirEntry, err error) error {
err := filepath.WalkDir(coverPath, func(path string, d fs.DirEntry, err error) error {
select {
case s := <-sig:
server.httpServer.Shutdown(context.TODO())
return fmt.Errorf("Signal: %v", s)
return fmt.Errorf("signal: %v", s)
default:
}
if d.IsDir() { // Only hash thumbnails for now
if d.IsDir() {
return nil
}
fmt.Println(len(server.readerQueue))
server.readerQueue <- path
return nil
})
elapsed := time.Now().Sub(start)
elapsed := time.Since(start)
fmt.Println("Err:", err, "local hashing took", elapsed)
select {
case s := <-sig:
server.httpServer.Shutdown(context.TODO())
log.Printf("Signal: %v", s)
}
s := <-sig
err = server.httpServer.Shutdown(context.TODO())
log.Printf("Signal: %v, error: %s", s, err)
}()
fmt.Println("Listening on ", server.httpServer.Addr)

View File

@ -46,9 +46,11 @@ func main() {
panic(err)
}
c.SortStrings(fileList)
var image []byte
var issue_id string
var files = []string{"ComicInfo.xml", fileList[0]}
var (
image []byte
issueID string
files = []string{"ComicInfo.xml", fileList[0]}
)
fmt.Printf("Extracting %s\n", fileList[0])
err = unrar.Extract(context.TODO(), file, files, func(ctx context.Context, f archiver.File) error {
r, err := f.Open()
@ -62,7 +64,7 @@ func main() {
}
parts := strings.Split(strings.TrimRight(ci.Web, "/"), "/")
ids := strings.Split(parts[len(parts)-1], "-")
issue_id = ids[1]
issueID = ids[1]
} else {
image, err = io.ReadAll(r)
if err != nil {
@ -75,7 +77,7 @@ func main() {
panic(err)
}
file.Close()
file, err = os.Create(*fileArchive + "." + issue_id + ".image")
file, err = os.Create(*fileArchive + "." + issueID + ".image")
if err != nil {
panic(err)
}

View File

@ -23,10 +23,9 @@ func init() {
// DisableBlockSmoothing: false,
// DCTMethod: jpeg.DCTFloat,
// })}, jpeg.DecodeConfig)
}
func save_image(im image.Image, name string) {
func saveImage(im image.Image, name string) {
file, err := os.Create(name)
if err != nil {
log.Printf("Failed to open file %s: %s", "tmp.png", err)
@ -63,18 +62,17 @@ func debugImage(im image.Image, width, height int) {
gray := goimagehash.ToGray(im, nil)
resized := goimagehash.Resize(gray, width, height, nil)
save_image(im, "go.rgb.png")
saveImage(im, "go.rgb.png")
log.Println("rgb")
log.Println(fmtImage(im))
save_image(gray, "go.gray.png")
saveImage(gray, "go.gray.png")
log.Println("gray")
log.Println(fmtImage(gray))
save_image(resized, "go.resized.png")
saveImage(resized, "go.resized.png")
log.Println("resized")
log.Println(fmtImage(resized))
}
func main() {

64
main.go
View File

@ -13,25 +13,25 @@ import (
)
const (
H_0 uint64 = 0b11111111 << (8 * iota)
H_1
H_2
H_3
H_4
H_5
H_6
H_7
H0 uint64 = 0b11111111 << (8 * iota)
H1
H2
H3
H4
H5
H6
H7
)
const (
Shift_0 = (8 * iota)
Shift_1
Shift_2
Shift_3
Shift_4
Shift_5
Shift_6
Shift_7
Shift0 = (8 * iota)
Shift1
Shift2
Shift3
Shift4
Shift5
Shift6
Shift7
)
type Source string
@ -67,20 +67,20 @@ type ImageHash struct {
Kind goimagehash.Kind
}
func Atleast(maxDistance int, search_hash uint64, hashes []uint64) []Match {
matching_hashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them
for _, stored_hash := range hashes {
distance := bits.OnesCount64(search_hash ^ stored_hash)
func Atleast(maxDistance int, searchHash uint64, hashes []uint64) []Match {
matchingHashes := make([]Match, 0, len(hashes)/2) // hope that we don't need all of them
for _, storedHash := range hashes {
distance := bits.OnesCount64(searchHash ^ storedHash)
if distance <= maxDistance {
matching_hashes = append(matching_hashes, Match{distance, stored_hash})
matchingHashes = append(matchingHashes, Match{distance, storedHash})
}
}
return matching_hashes
return matchingHashes
}
func Insert[S ~[]E, E cmp.Ordered](slice S, item E) S {
index, item_found := slices.BinarySearch(slice, item)
if item_found {
index, itemFound := slices.BinarySearch(slice, item)
if itemFound {
return slice
}
return slices.Insert(slice, index, item)
@ -133,14 +133,14 @@ func HashImage(i Im) Hash {
func SplitHash(hash uint64) [8]uint8 {
return [8]uint8{
uint8((hash & H_7) >> Shift_7),
uint8((hash & H_6) >> Shift_6),
uint8((hash & H_5) >> Shift_5),
uint8((hash & H_4) >> Shift_4),
uint8((hash & H_3) >> Shift_3),
uint8((hash & H_2) >> Shift_2),
uint8((hash & H_1) >> Shift_1),
uint8((hash & H_0) >> Shift_0),
uint8((hash & H7) >> Shift7),
uint8((hash & H6) >> Shift6),
uint8((hash & H5) >> Shift5),
uint8((hash & H4) >> Shift4),
uint8((hash & H3) >> Shift3),
uint8((hash & H2) >> Shift2),
uint8((hash & H1) >> Shift1),
uint8((hash & H0) >> Shift0),
}
}