lordwelch/comic-hasher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: lordwelch:da54b3a4540f8622831a123410cb76eaa6e91b19
Branches Tags
lordwelch:main
..
compare: lordwelch:095c78f0e7bb1e7d95465da3d382dde7899dd76b
Branches Tags
lordwelch:main

No commits in common. "da54b3a4540f8622831a123410cb76eaa6e91b19" and "095c78f0e7bb1e7d95465da3d382dde7899dd76b" have entirely different histories.
da54b3a454 ... 095c78f0e7

5 changed files with 31 additions and 39 deletions
Show Stats Expand all files Collapse all files

37
BasicMap.go
View File

@ -2,7 +2,6 @@ package ch
import ( import (
"cmp" "cmp"
"errors"
"fmt" "fmt"
"math/bits" "math/bits"
"slices" "slices"
@ -89,7 +88,7 @@ func (b *basicMapStorage) InsertHash(hashType int, hash uint64, ids *[]ID) {
if hashFound { if hashFound {
return return
} }
b.hashes[hashType] = slices.Insert(b.hashes[hashType], index, structHash{hash, ids}) slices.Insert(b.hashes[hashType], index, structHash{hash, ids})
} }
func (b *basicMapStorage) MapHashes(hash ImageHash) { func (b *basicMapStorage) MapHashes(hash ImageHash) {
@ -100,7 +99,6 @@ func (b *basicMapStorage) MapHashes(hash ImageHash) {
ids, ok := b.ids[hash.ID] ids, ok := b.ids[hash.ID]
if !ok { if !ok {
ids = &[]ID{hash.ID} ids = &[]ID{hash.ID}
b.ids[hash.ID] = ids
} }
b.InsertHash(hashType, ih.Hash, ids) b.InsertHash(hashType, ih.Hash, ids)
@ -112,9 +110,6 @@ func (b *basicMapStorage) DecodeHashes(hashes SavedHashes) error {
b.hashes[hashType] = make([]structHash, len(sourceHashes)) b.hashes[hashType] = make([]structHash, len(sourceHashes))
for savedHash, idlistLocation := range sourceHashes { for savedHash, idlistLocation := range sourceHashes {
b.hashes[hashType] = append(b.hashes[hashType], structHash{savedHash, &hashes.IDs[idlistLocation]}) b.hashes[hashType] = append(b.hashes[hashType], structHash{savedHash, &hashes.IDs[idlistLocation]})
for _, id := range hashes.IDs[idlistLocation] {
b.ids[id] = &hashes.IDs[idlistLocation]
}
} }
} }
for hashType := range b.hashes { for hashType := range b.hashes {
@ -134,23 +129,15 @@ func (b *basicMapStorage) printSizes() {
} }
func (b *basicMapStorage) EncodeHashes() (SavedHashes, error) { func (b *basicMapStorage) EncodeHashes() (SavedHashes, error) {
hashes := SavedHashes{ hashes := SavedHashes{}
Hashes: [3]map[uint64]int{
make(map[uint64]int),
make(map[uint64]int),
make(map[uint64]int),
},
}
idmap := map[*[]ID]int{} idmap := map[*[]ID]int{}
for _, ids := range b.ids { for _, ids := range b.ids {
if _, ok := idmap[ids]; ok { if _, ok := idmap[ids]; ok {
continue continue
} }
idmap[ids] = len(hashes.IDs)
hashes.IDs = append(hashes.IDs, *ids) hashes.IDs = append(hashes.IDs, *ids)
idmap[ids] = len(hashes.IDs)
} }
for hashType, hashToID := range b.hashes { for hashType, hashToID := range b.hashes {
for _, hash := range hashToID { for _, hash := range hashToID {
hashes.Hashes[hashType][hash.hash] = idmap[hash.ids] hashes.Hashes[hashType][hash.hash] = idmap[hash.ids]
@ -159,22 +146,22 @@ func (b *basicMapStorage) EncodeHashes() (SavedHashes, error) {
return hashes, nil return hashes, nil
} }
func (b *basicMapStorage) AssociateIDs(newids []NewIDs) error { func (b *basicMapStorage) AssociateIDs(newids []NewIDs) {
for _, newid := range newids { for _, newid := range newids {
ids, found := b.ids[newid.OldID] ids, found := b.ids[newid.OldID]
if !found { if !found {
msg := "No IDs belonging to " + string(newid.OldID.Domain) + " exist on this server" msg := "No IDs belonging to " + newid.OldID.Domain + "exist on this server"
return errors.New(msg) panic(msg)
} }
*ids = InsertID(*ids, newid.NewID) *ids = InsertID(*ids, newid.NewID)
} }
return nil
} }
func (b *basicMapStorage) GetIDs(id ID) IDList { func (b *basicMapStorage) GetIDs(id ID) IDList {
ids, found := b.ids[id] ids, found := b.ids[id]
if !found { if !found {
return nil msg := "No IDs belonging to " + id.Domain + "exist on this server"
panic(msg)
} }
return ToIDList(*ids) return ToIDList(*ids)
} }
@ -182,8 +169,12 @@ func (b *basicMapStorage) GetIDs(id ID) IDList {
func NewBasicMapStorage() (HashStorage, error) { func NewBasicMapStorage() (HashStorage, error) {
storage := &basicMapStorage{ storage := &basicMapStorage{
hashMutex: sync.RWMutex{}, hashMutex: sync.RWMutex{},
ids: make(map[ID]*[]ID),
hashes: [3][]structHash{}, hashes: [3][]structHash{
[]structHash{},
[]structHash{},
[]structHash{},
},
} }
return storage, nil return storage, nil
} }

9
cmd/comic-hasher/main.go
View File

@ -459,7 +459,7 @@ func (s *Server) addCover(w http.ResponseWriter, r *http.Request) {
return return
default: default:
} }
s.hashingQueue <- ch.Im{Im: i, Format: format, ID: ch.ID{Domain: ch.Source(domain), ID: ID}} s.hashingQueue <- ch.Im{Im: i, Format: format, ID: ch.ID{Domain: ch.Source(domain), ID: ID}, Path: ""}
writeJson(w, http.StatusOK, result{Msg: "Success"}) writeJson(w, http.StatusOK, result{Msg: "Success"})
} }
@ -489,7 +489,7 @@ func (s *Server) hasher(workerID int, done func()) {
} }
elapsed := time.Since(start) elapsed := time.Since(start)
log.Printf("Hashing took %v: worker: %v. %s: %064b id: %s\n", elapsed, workerID, hash.Hashes[0].Kind, hash.Hashes[0].Hash, hash.ID) log.Printf("Hashing took %v: worker: %v. path: %s %s: %064b id: %s\n", elapsed, workerID, image.Path, hash.Hashes[0].Kind, hash.Hashes[0].Hash, hash.ID)
} }
} }
@ -509,6 +509,7 @@ func (s *Server) reader(workerID int, done func()) {
im := ch.Im{ im := ch.Im{
Im: i, Format: format, Im: i, Format: format,
ID: ch.ID{Domain: ch.Source(filepath.Base(filepath.Dir(filepath.Dir(path)))), ID: filepath.Base(filepath.Dir(path))}, ID: ch.ID{Domain: ch.Source(filepath.Base(filepath.Dir(filepath.Dir(path)))), ID: filepath.Base(filepath.Dir(path))},
Path: path,
} }
select { select {
case <-s.quit: case <-s.quit:
@ -552,8 +553,8 @@ func (s *Server) DecodeHashes(format Format, hashes []byte) error {
} }
loadedHashes := ch.SavedHashes{} loadedHashes := ch.SavedHashes{}
err := decoder(hashes, &loadedHashes) err := decoder(hashes, &loadedHashes)
if err != nil { if err != nil || len(loadedHashes.IDs) == 0 {
fmt.Println("Failed to load hashes, checking if they are old hashes", format, ":", err) fmt.Println("Failed to load hashes, checking if they are old hashes", err)
oldHashes := make(ch.OldSavedHashes) oldHashes := make(ch.OldSavedHashes)
if err = decoder(hashes, &oldHashes); err != nil { if err = decoder(hashes, &oldHashes); err != nil {
return err return err

3
hashing.go
View File

@ -63,6 +63,7 @@ type Result struct {
type Im struct { type Im struct {
Im image.Image Im image.Image
Format string Format string
Path string
ID ID ID ID
} }
@ -163,7 +164,7 @@ type HashStorage interface {
MapHashes(ImageHash) MapHashes(ImageHash)
DecodeHashes(hashes SavedHashes) error DecodeHashes(hashes SavedHashes) error
EncodeHashes() (SavedHashes, error) EncodeHashes() (SavedHashes, error)
AssociateIDs(newIDs []NewIDs) error AssociateIDs(newIDs []NewIDs)
GetIDs(id ID) IDList GetIDs(id ID) IDList
} }

15
sqlite.go
View File

@ -354,26 +354,26 @@ func (s *sqliteStorage) EncodeHashes() (SavedHashes, error) {
return hashes, nil return hashes, nil
} }
func (s *sqliteStorage) AssociateIDs(newIDs []NewIDs) error { func (s *sqliteStorage) AssociateIDs(newIDs []NewIDs) {
for _, ids := range newIDs { for _, ids := range newIDs {
var oldIDID, newIDID int var oldIDID, newIDID int
_, err := s.db.Exec(`INSERT INTO IDs domain,id VALUES (?,?)`, ids.NewID.Domain, ids.NewID.ID) _, err := s.db.Exec(`INSERT INTO IDs domain,id VALUES (?,?)`, ids.NewID.Domain, ids.NewID.ID)
if err != nil { if err != nil {
return err panic(err)
} }
rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.NewID.Domain, ids.NewID.ID) rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.NewID.Domain, ids.NewID.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) { if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err panic(err)
} }
if rows.Next() { if rows.Next() {
rows.Scan(&newIDID) rows.Scan(&newIDID)
} else { } else {
return errors.New("Unable to insert New ID into database") panic("Unable to insert New ID into database")
} }
rows.Close() rows.Close()
rows, err = s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.OldID.Domain, ids.OldID.ID) rows, err = s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.OldID.Domain, ids.OldID.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) { if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err panic(err)
} }
if rows.Next() { if rows.Next() {
rows.Scan(&oldIDID) rows.Scan(&oldIDID)
@ -382,17 +382,16 @@ func (s *sqliteStorage) AssociateIDs(newIDs []NewIDs) error {
} }
_, err = s.db.Exec(`INSERT INTO id_hash (hashid, id_id) SELECT hashid,? FROM id_hash where id_id=?`, newIDID, oldIDID) _, err = s.db.Exec(`INSERT INTO id_hash (hashid, id_id) SELECT hashid,? FROM id_hash where id_id=?`, newIDID, oldIDID)
if err != nil { if err != nil {
return fmt.Errorf("Unable to associate IDs: %w", err) panic(err)
} }
} }
return nil
} }
func (s *sqliteStorage) GetIDs(id ID) IDList { func (s *sqliteStorage) GetIDs(id ID) IDList {
var idid int var idid int
rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, id.Domain, id.ID) rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, id.Domain, id.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) { if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil panic(err)
} }
if rows.Next() { if rows.Next() {
rows.Scan(&idid) rows.Scan(&idid)

4
vp-tree.go
View File

@ -85,8 +85,8 @@ func (v *VPTree) EncodeHashes() (SavedHashes, error) {
return SavedHashes{}, errors.New("Not Implemented") return SavedHashes{}, errors.New("Not Implemented")
} }
func (v *VPTree) AssociateIDs(newIDs []NewIDs) error { func (v *VPTree) AssociateIDs(newIDs []NewIDs) {
return errors.New("Not Implemented") panic("Not Implemented")
} }
func (v *VPTree) GetIDs(id ID) IDList { func (v *VPTree) GetIDs(id ID) IDList {