Re-implement sqlite
This commit is contained in:
Timmy Welch
parent
1f37684862
commit
a2765b0582
575
sqlite.go
575
sqlite.go
@ -7,140 +7,102 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math/bits"
|
||||
"strings"
|
||||
|
||||
"gitea.narnian.us/lordwelch/goimagehash"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type sqliteStorage struct {
|
||||
db *sql.DB
|
||||
}
|
||||
type sqliteHash struct {
|
||||
hashid int
|
||||
Result
|
||||
|
||||
hashExactMatchStatement *sql.Stmt
|
||||
hashPartialMatchStatement *sql.Stmt
|
||||
|
||||
idMatchStatement *sql.Stmt
|
||||
|
||||
insertHash *sql.Stmt
|
||||
insertID *sql.Stmt
|
||||
insertEID *sql.Stmt
|
||||
insertIEID *sql.Stmt
|
||||
idExists *sql.Stmt
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) findExactHashes(statement *sql.Stmt, items ...interface{}) ([]sqliteHash, error) { // exact matches are also found by partial matches. Don't bother with exact matches so we don't have to de-duplicate
|
||||
hashes := []sqliteHash{}
|
||||
rows, err := statement.Query(items...)
|
||||
func (s *sqliteStorage) findExactHashes(statement *sql.Stmt, hash Hash) ([]ID, error) {
|
||||
if statement == nil {
|
||||
statement = s.hashExactMatchStatement
|
||||
}
|
||||
hashes := []ID{}
|
||||
rows, err := statement.Query(hash.Kind, int64(hash.Hash))
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
r = sqliteHash{Result: Result{IDs: make(IDList)}}
|
||||
h int64
|
||||
r = ID{}
|
||||
)
|
||||
err = rows.Scan(&r.hashid, &h, &r.Hash.Kind)
|
||||
err = rows.Scan(&r.Domain, &r.ID)
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
return hashes, err
|
||||
}
|
||||
r.Hash.Hash = uint64(h)
|
||||
hashes = append(hashes, r)
|
||||
}
|
||||
rows.Close()
|
||||
statement, err = s.db.PrepareContext(context.Background(), `SELECT IDS.domain, IDs.id FROM IDs JOIN id_hash ON IDs.rowid = id_hash.idid WHERE (id_hash.hashid=?) ORDER BY IDs.domain, IDs.ID;`)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
for _, hash := range hashes {
|
||||
rows, err := statement.Query(hash.hashid)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var source Source
|
||||
var id string
|
||||
err := rows.Scan(&source, &id)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
hash.IDs[source] = append(hash.IDs[source], id)
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
return hashes, nil
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) findPartialHashes(tl timeLog, max int, search_hash int64, kind goimagehash.Kind) ([]sqliteHash, error) { // exact matches are also found by partial matches. Don't bother with exact matches so we don't have to de-duplicate
|
||||
hashes := []sqliteHash{}
|
||||
statement, err := s.db.PrepareContext(context.Background(), `SELECT rowid,hash,kind FROM Hashes WHERE (kind=?) AND (((hash >> (0 * 8) & 0xFF)=(?2 >> (0 * 8) & 0xFF)) OR ((hash >> (1 * 8) & 0xFF)=(?2 >> (1 * 8) & 0xFF)) OR ((hash >> (2 * 8) & 0xFF)=(?2 >> (2 * 8) & 0xFF)) OR ((hash >> (3 * 8) & 0xFF)=(?2 >> (3 * 8) & 0xFF)) OR ((hash >> (4 * 8) & 0xFF)=(?2 >> (4 * 8) & 0xFF)) OR ((hash >> (5 * 8) & 0xFF)=(?2 >> (5 * 8) & 0xFF)) OR ((hash >> (6 * 8) & 0xFF)=(?2 >> (6 * 8) & 0xFF)) OR ((hash >> (7 * 8) & 0xFF)=(?2 >> (7 * 8) & 0xFF)));`)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
func (s *sqliteStorage) findPartialHashes(tl timeLog, statement *sql.Stmt, max int, hash Hash) ([]Result, error) {
|
||||
if statement == nil {
|
||||
statement = s.hashPartialMatchStatement
|
||||
}
|
||||
rows, err := statement.Query(kind, int64(search_hash))
|
||||
hashResults := []Result{}
|
||||
rows, err := statement.Query(hash.Kind, int64(hash.Hash))
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
return hashResults, err
|
||||
}
|
||||
|
||||
results := map[Hash][]ID{}
|
||||
for rows.Next() {
|
||||
var (
|
||||
r = sqliteHash{Result: Result{IDs: make(IDList)}}
|
||||
h int64
|
||||
tmpHash int64
|
||||
sqlHash = Hash{Kind: hash.Kind}
|
||||
id ID
|
||||
)
|
||||
err = rows.Scan(&r.hashid, &h, &r.Hash.Kind)
|
||||
err = rows.Scan(&tmpHash, &id.Domain, &id.ID)
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
return hashes, err
|
||||
return hashResults, err
|
||||
}
|
||||
r.Hash.Hash = uint64(h)
|
||||
r.Distance = bits.OnesCount64(uint64(search_hash) ^ r.Hash.Hash)
|
||||
if r.Distance <= max {
|
||||
hashes = append(hashes, r)
|
||||
sqlHash.Hash = uint64(tmpHash)
|
||||
results[sqlHash] = append(results[sqlHash], id)
|
||||
}
|
||||
for sqlHash, ids := range results {
|
||||
res := Result{
|
||||
Hash: sqlHash,
|
||||
Distance: bits.OnesCount64(hash.Hash ^ sqlHash.Hash),
|
||||
}
|
||||
if res.Distance <= max {
|
||||
res.IDs = ToIDList(ids)
|
||||
hashResults = append(hashResults, res)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
tl.logTime("Filter partial " + kind.String())
|
||||
|
||||
statement, err = s.db.PrepareContext(context.Background(), `SELECT DISTINCT IDS.domain, IDs.id, id_hash.hashid FROM IDs JOIN id_hash ON IDs.rowid = id_hash.idid WHERE (id_hash.hashid in (`+strings.TrimRight(strings.Repeat("?,", len(hashes)), ",")+`)) ORDER BY IDs.domain, IDs.ID;`)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
|
||||
var ids []any
|
||||
for _, hash := range hashes {
|
||||
ids = append(ids, hash.hashid)
|
||||
}
|
||||
rows, err = statement.Query(ids...)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var source Source
|
||||
var id string
|
||||
var hashid int
|
||||
err := rows.Scan(&source, &id, &hashid)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
for _, hash := range hashes {
|
||||
if hash.hashid == hashid {
|
||||
hash.IDs[source] = append(hash.IDs[source], id)
|
||||
}
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
return hashes, nil
|
||||
return hashResults, nil
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) dropIndexes() error {
|
||||
_, err := s.db.Exec(`
|
||||
DROP INDEX IF EXISTS hash_index;
|
||||
DROP INDEX IF EXISTS hash_1_index;
|
||||
DROP INDEX IF EXISTS hash_2_index;
|
||||
DROP INDEX IF EXISTS hash_3_index;
|
||||
DROP INDEX IF EXISTS hash_4_index;
|
||||
DROP INDEX IF EXISTS hash_5_index;
|
||||
DROP INDEX IF EXISTS hash_6_index;
|
||||
DROP INDEX IF EXISTS hash_7_index;
|
||||
DROP INDEX IF EXISTS hash_8_index;
|
||||
|
||||
DROP INDEX IF EXISTS hash_index;
|
||||
DROP INDEX IF EXISTS hash_1_index;
|
||||
DROP INDEX IF EXISTS hash_2_index;
|
||||
DROP INDEX IF EXISTS hash_3_index;
|
||||
DROP INDEX IF EXISTS hash_4_index;
|
||||
DROP INDEX IF EXISTS hash_5_index;
|
||||
DROP INDEX IF EXISTS hash_6_index;
|
||||
DROP INDEX IF EXISTS hash_7_index;
|
||||
DROP INDEX IF EXISTS hash_8_index;
|
||||
|
||||
DROP INDEX IF EXISTS id_domain;
|
||||
`)
|
||||
DROP INDEX IF EXISTS id_domain;
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -149,21 +111,20 @@ func (s *sqliteStorage) dropIndexes() error {
|
||||
|
||||
func (s *sqliteStorage) createIndexes() error {
|
||||
_, err := s.db.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS hash_index ON Hashes (kind, hash);
|
||||
CREATE INDEX IF NOT EXISTS hash_1_index ON Hashes ((hash >> (0 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_2_index ON Hashes ((hash >> (1 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_3_index ON Hashes ((hash >> (2 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_4_index ON Hashes ((hash >> (3 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_5_index ON Hashes ((hash >> (4 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_6_index ON Hashes ((hash >> (5 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_7_index ON Hashes ((hash >> (6 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_8_index ON Hashes ((hash >> (7 * 8) & 0xFF));
|
||||
|
||||
CREATE INDEX IF NOT EXISTS hash_index ON Hashes (kind, hash);
|
||||
CREATE INDEX IF NOT EXISTS hash_1_index ON Hashes ((hash >> (0 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_2_index ON Hashes ((hash >> (1 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_3_index ON Hashes ((hash >> (2 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_4_index ON Hashes ((hash >> (3 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_5_index ON Hashes ((hash >> (4 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_6_index ON Hashes ((hash >> (5 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_7_index ON Hashes ((hash >> (6 * 8) & 0xFF));
|
||||
CREATE INDEX IF NOT EXISTS hash_8_index ON Hashes ((hash >> (7 * 8) & 0xFF));
|
||||
|
||||
CREATE INDEX IF NOT EXISTS id_domain ON IDs (domain, id);
|
||||
PRAGMA shrink_memory;
|
||||
ANALYZE;
|
||||
`)
|
||||
CREATE INDEX IF NOT EXISTS id_domain ON IDs (domain, stringid);
|
||||
PRAGMA shrink_memory;
|
||||
ANALYZE;
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -178,47 +139,33 @@ func (s *sqliteStorage) GetMatches(hashes []Hash, max int, exactOnly bool) ([]Re
|
||||
tl.resetTime()
|
||||
|
||||
if exactOnly { // exact matches are also found by partial matches. Don't bother with exact matches so we don't have to de-duplicate
|
||||
|
||||
statement, err := s.db.Prepare(`SELECT rowid,hash,kind FROM Hashes WHERE ` + strings.TrimSuffix(strings.Repeat("(hash=? AND kind=?) OR", len(hashes)), "OR") + `ORDER BY kind,hash;`)
|
||||
if err != nil {
|
||||
tl.logTime("Fail exact")
|
||||
return foundMatches, err
|
||||
}
|
||||
|
||||
args := make([]interface{}, 0, len(hashes)*2)
|
||||
for _, hash := range hashes {
|
||||
if hash.Hash != 0 {
|
||||
args = append(args, int64(hash.Hash), hash.Kind)
|
||||
idlist, err := s.findExactHashes(nil, hash)
|
||||
if err != nil {
|
||||
return foundMatches, err
|
||||
}
|
||||
}
|
||||
hashes, err := s.findExactHashes(statement, args...)
|
||||
if err != nil {
|
||||
return foundMatches, err
|
||||
}
|
||||
for _, hash := range hashes {
|
||||
foundMatches = append(foundMatches, hash.Result)
|
||||
foundMatches = append(foundMatches, Result{
|
||||
IDs: ToIDList(idlist),
|
||||
Hash: hash,
|
||||
})
|
||||
}
|
||||
|
||||
// If we have exact matches don't bother with other matches
|
||||
if len(foundMatches) > 0 && exactOnly {
|
||||
return foundMatches, nil
|
||||
}
|
||||
tl.logTime("Search Exact")
|
||||
return foundMatches, nil
|
||||
}
|
||||
|
||||
foundHashes := make(map[uint64]struct{})
|
||||
|
||||
for _, hash := range hashes {
|
||||
hashes, err := s.findPartialHashes(tl, max, int64(hash.Hash), hash.Kind)
|
||||
results, err := s.findPartialHashes(tl, nil, max, hash)
|
||||
if err != nil {
|
||||
return foundMatches, err
|
||||
}
|
||||
tl.logTime("Search partial " + hash.Kind.String())
|
||||
tl.logTime(fmt.Sprintf("Search partial %v", hash.Kind))
|
||||
|
||||
for _, hash := range hashes {
|
||||
for _, hash := range results {
|
||||
if _, alreadyMatched := foundHashes[hash.Hash.Hash]; !alreadyMatched {
|
||||
foundHashes[hash.Hash.Hash] = struct{}{}
|
||||
foundMatches = append(foundMatches, hash.Result)
|
||||
foundMatches = append(foundMatches, hash)
|
||||
} else {
|
||||
log.Println("Hash already found", hash)
|
||||
}
|
||||
@ -228,17 +175,15 @@ func (s *sqliteStorage) GetMatches(hashes []Hash, max int, exactOnly bool) ([]Re
|
||||
return foundMatches, nil
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) MapHashes(hash ImageHash) {
|
||||
tx, err := s.db.BeginTx(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
insertHashes, err := tx.Prepare(`INSERT INTO Hashes (hash,kind) VALUES (?,?) ON CONFLICT DO UPDATE SET hash=?1 RETURNING hashid`)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
func (s *sqliteStorage) mapHashes(tx *sql.Tx, hash ImageHash) {
|
||||
var err error
|
||||
insertHash := tx.Stmt(s.insertHash)
|
||||
insertID := tx.Stmt(s.insertID)
|
||||
idExists := tx.Stmt(s.idExists)
|
||||
insertEID := tx.Stmt(s.insertEID)
|
||||
insertIEID := tx.Stmt(s.insertIEID)
|
||||
|
||||
rows, err := tx.Query(`INSERT INTO IDs (domain,id) VALUES (?,?) ON CONFLICT DO UPDATE SET domain=?1 RETURNING idid`, hash.ID.Domain, hash.ID.ID)
|
||||
rows, err := insertID.Query(hash.ID.Domain, hash.ID.ID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@ -253,54 +198,93 @@ func (s *sqliteStorage) MapHashes(hash ImageHash) {
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
hash_ids := []int64{}
|
||||
for _, hash := range hash.Hashes {
|
||||
rows, err := insertHashes.Query(int64(hash.Hash), hash.Kind)
|
||||
_, err := insertHash.Exec(hash.Kind, int64(hash.Hash), id_id)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if !rows.Next() {
|
||||
panic("Unable to insert Hash")
|
||||
}
|
||||
|
||||
var id int64
|
||||
err = rows.Scan(&id)
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
hash_ids = append(hash_ids, id)
|
||||
}
|
||||
var ids []any = make([]any, 0, len(hash_ids)+1)
|
||||
ids = append(ids, id_id)
|
||||
for _, hash_id := range hash_ids {
|
||||
ids = append(ids, hash_id)
|
||||
}
|
||||
_, err = tx.Exec(`INSERT INTO id_hash (idid, hashid) VALUES `+strings.TrimSuffix(strings.Repeat("(?1, ?),", len(hash_ids)), ",")+` ON CONFLICT DO NOTHING;`, ids...)
|
||||
rows.Close()
|
||||
row := idExists.QueryRow(id_id)
|
||||
var count int64
|
||||
err = row.Scan(&count)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("Failed inserting: %v,%v: %w", hash.ID.Domain, hash.ID.ID, err))
|
||||
panic(fmt.Errorf("failed to query id: %w", err))
|
||||
}
|
||||
if count < 1 {
|
||||
row := insertEID.QueryRow()
|
||||
var eid int64
|
||||
err = row.Scan(&eid)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to insert equivalent id: %w", err))
|
||||
}
|
||||
_, err := insertIEID.Exec(id_id, eid)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to associate equivalent IDs: %w", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *sqliteStorage) MapHashes(hash ImageHash) {
|
||||
tx, err := s.db.BeginTx(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s.mapHashes(tx, hash)
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
insertHashes.Close()
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) DecodeHashes(hashes SavedHashes) error {
|
||||
return nil
|
||||
err := s.dropIndexes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.BeginTx(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
insertID := tx.Stmt(s.insertID)
|
||||
insertEID := tx.Stmt(s.insertEID)
|
||||
insertIEID := tx.Stmt(s.insertIEID)
|
||||
for _, idlist := range hashes.IDs {
|
||||
var eid int64
|
||||
id_ids := make([]int64, 0, len(idlist))
|
||||
for _, id := range idlist {
|
||||
var id_id int64
|
||||
row := insertID.QueryRow(id.Domain, id.ID)
|
||||
err = row.Scan(&id_id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert id: %w", err)
|
||||
}
|
||||
id_ids = append(id_ids, id_id)
|
||||
}
|
||||
row := insertEID.QueryRow()
|
||||
err = row.Scan(&eid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert equivalent id: %w", err)
|
||||
}
|
||||
for _, id_id := range id_ids {
|
||||
_, err = insertIEID.Exec(id_id, eid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, savedHash := range hashes.Hashes {
|
||||
s.MapHashes(ImageHash{
|
||||
s.mapHashes(tx, ImageHash{
|
||||
Hashes: []Hash{savedHash.Hash},
|
||||
ID: savedHash.ID,
|
||||
})
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = s.createIndexes()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -310,97 +294,180 @@ func (s *sqliteStorage) DecodeHashes(hashes SavedHashes) error {
|
||||
|
||||
func (s *sqliteStorage) EncodeHashes() (SavedHashes, error) {
|
||||
hashes := SavedHashes{}
|
||||
conn, err := s.db.Conn(context.Background())
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
defer conn.Close()
|
||||
rows, err := conn.QueryContext(context.Background(), "SELECT IDs.domain,IDs.id,Hashes.hash,Hashes.kind FROM Hashes JOIN id_hash ON id_hash.hashid = hashes.rowid JOIN IDs ON IDs.rowid = id_hash.idid ORDER BY IDs.ID,Hashes.kind,Hashes.hash;")
|
||||
|
||||
rows, err := tx.Query("SELECT Hashes.kind, Hashes.hash, IDs.domain, IDs.stringid FROM Hashes JOIN IDs ON Hashes.id=IDs.id ORDER BY Hashes.kind, Hashes.hash;")
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var (
|
||||
hash SavedHash
|
||||
tmpHash int64
|
||||
)
|
||||
err = rows.Scan(&hash.Hash.Kind, &tmpHash, &hash.ID.Domain, &hash.ID.ID)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
hash.Hash.Hash = uint64(tmpHash)
|
||||
hashes.InsertHash(hash)
|
||||
}
|
||||
rows, err = tx.Query("SELECT IEIDs.equivalentid, IDs.domain, IDs.stringid FROM IDs JOIN IDsToEquivalantIDs AS IEIDs ON IDs.id=IEIDs.idid ORDER BY IEIDs.equivalentid, IDs.domain, IDs.stringid;")
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
return hashes, err
|
||||
}
|
||||
var (
|
||||
id ID
|
||||
hash Hash
|
||||
previousEid int64 = -1
|
||||
ids []ID
|
||||
)
|
||||
err = rows.Scan(&id.Domain, &id.ID, &hash.Hash, &hash.Kind)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
for rows.Next() {
|
||||
var (
|
||||
id ID
|
||||
newEid int64
|
||||
)
|
||||
err = rows.Scan(&newEid, &id.Domain, &id.Domain)
|
||||
if err != nil {
|
||||
return hashes, err
|
||||
}
|
||||
if newEid != previousEid {
|
||||
previousEid = newEid
|
||||
// Only keep groups len>1 as they are mapped in SavedHashes.Hashes
|
||||
if len(ids) > 1 {
|
||||
hashes.IDs = append(hashes.IDs, ids)
|
||||
}
|
||||
ids = make([]ID, 0)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
hashes.InsertHash(hash, id)
|
||||
|
||||
return hashes, nil
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) AssociateIDs(newIDs []NewIDs) error {
|
||||
tx, err := s.db.BeginTx(context.Background(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
insertID := tx.Stmt(s.insertID)
|
||||
insertIEID := tx.Stmt(s.insertIEID)
|
||||
for _, ids := range newIDs {
|
||||
var oldIDID, newIDID int
|
||||
_, err := s.db.Exec(`INSERT INTO IDs domain,id VALUES (?,?)`, ids.NewID.Domain, ids.NewID.ID)
|
||||
var (
|
||||
newRowid int64
|
||||
oldRowid int64
|
||||
eid int64
|
||||
)
|
||||
rows := tx.QueryRow(`SELECT ITEI.idid, ITEI.equivalentid from IDs JOIN IDsToEquivalantIDs AS ITEI ON IDs.id=ITEI.idid WHERE domain=? AND stringid=?`, ids.OldID.Domain, ids.OldID.ID)
|
||||
|
||||
err := rows.Scan(&oldRowid, &eid)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrIDNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
rows = insertID.QueryRow(ids.NewID.Domain, ids.NewID.ID)
|
||||
|
||||
err = rows.Scan(&newRowid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.NewID.Domain, ids.NewID.ID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if rows.Next() {
|
||||
rows.Scan(&newIDID)
|
||||
} else {
|
||||
return errors.New("Unable to insert New ID into database")
|
||||
}
|
||||
rows.Close()
|
||||
rows, err = s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, ids.OldID.Domain, ids.OldID.ID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if rows.Next() {
|
||||
rows.Scan(&oldIDID)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
_, err = s.db.Exec(`INSERT INTO id_hash (hashid, id_id) SELECT hashid,? FROM id_hash where id_id=?`, newIDID, oldIDID)
|
||||
_, err = insertIEID.Exec(newRowid, eid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to associate IDs: %w", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) GetIDs(id ID) IDList {
|
||||
var idid int
|
||||
rows, err := s.db.Query(`SELECT idid FROM IDs WHERE domain=? AND id=?`, id.Domain, id.ID)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
var ids []ID
|
||||
rows, err := s.idMatchStatement.Query(id.Domain, id.ID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if rows.Next() {
|
||||
rows.Scan(&idid)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
rows, err = s.db.Query(`SELECT id_hash FROM id_hash WHERE id_id=?`, idid)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
panic(err)
|
||||
}
|
||||
var hashIDs []interface{}
|
||||
for rows.Next() {
|
||||
var hashID int
|
||||
rows.Scan(&hashID)
|
||||
hashIDs = append(hashIDs, hashID)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
IDs := make(IDList)
|
||||
rows, err = s.db.Query(`SELECT IDs.domain,IDs.id FROM id_hash JOIN IDs ON id_hash.idid==IDs.idid WHERE hash_id in (`+strings.TrimRight(strings.Repeat("?,", len(hashIDs)), ",")+`)`, hashIDs...)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
panic(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var id ID
|
||||
rows.Scan(&id.Domain, id.ID)
|
||||
IDs[id.Domain] = append(IDs[id.Domain], id.ID)
|
||||
err = rows.Scan(&id.Domain, &id.ID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return IDs
|
||||
return ToIDList(ids)
|
||||
}
|
||||
|
||||
func (s *sqliteStorage) PrepareStatements() error {
|
||||
var err error
|
||||
s.insertHash, err = s.db.Prepare(`INSERT INTO Hashes (kind, hash, id) VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET kind=?1`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.insertID, err = s.db.Prepare(`INSERT INTO IDs (domain, stringid) VALUES (?,?) ON CONFLICT DO UPDATE SET domain=?1 RETURNING id`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.insertEID, err = s.db.Prepare(`INSERT INTO EquivalentIDs DEFAULT VALUES RETURNING id;`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.insertIEID, err = s.db.Prepare(`INSERT INTO IDsToEquivalantIDs (idid, equivalentid) VALUES (?, ?);`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.idExists, err = s.db.Prepare(`SELECT COUNT(*) from IDsToEquivalantIDs WHERE idid=?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.hashExactMatchStatement, err = s.db.Prepare(`
|
||||
select IDs.domain, IDs.stringid from IDs
|
||||
join IDsToEquivalantIDs as IEIDs on IDs.id=IEIDs.idid
|
||||
join (
|
||||
select QEIDs.id as id from EquivalentIDs as QEIDs
|
||||
join IDsToEquivalantIDs as QIEIDs on QEIDs.id=QIEIDs.equivalentid
|
||||
join IDs as QIDs on QIDs.id=QIEIDs.idid
|
||||
join Hashes on Hashes.id=QIDs.id
|
||||
where (Hashes.kind=? AND Hashes.hash=?)
|
||||
) as EIDs on EIDs.id=IEIDs.equivalentid;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.hashPartialMatchStatement, err = s.db.Prepare(`
|
||||
select EIDs.hash, IDs.domain, IDs.stringid from IDs
|
||||
join IDsToEquivalantIDs as IEIDs on IDs.id=IEIDs.idid
|
||||
join (
|
||||
select Hashes.hash as hash, QEIDs.id as id from EquivalentIDs as QEIDs
|
||||
join IDsToEquivalantIDs as QIEIDs on QEIDs.id=QIEIDs.equivalentid
|
||||
join IDs as QIDs on QIDs.id=QIEIDs.idid
|
||||
join Hashes on Hashes.id=QIDs.id
|
||||
where (Hashes.kind=? AND (((Hashes.hash >> (0 * 8) & 0xFF)=(?2 >> (0 * 8) & 0xFF)) OR ((Hashes.hash >> (1 * 8) & 0xFF)=(?2 >> (1 * 8) & 0xFF)) OR ((Hashes.hash >> (2 * 8) & 0xFF)=(?2 >> (2 * 8) & 0xFF)) OR ((Hashes.hash >> (3 * 8) & 0xFF)=(?2 >> (3 * 8) & 0xFF)) OR ((Hashes.hash >> (4 * 8) & 0xFF)=(?2 >> (4 * 8) & 0xFF)) OR ((Hashes.hash >> (5 * 8) & 0xFF)=(?2 >> (5 * 8) & 0xFF)) OR ((Hashes.hash >> (6 * 8) & 0xFF)=(?2 >> (6 * 8) & 0xFF)) OR ((Hashes.hash >> (7 * 8) & 0xFF)=(?2 >> (7 * 8) & 0xFF))))
|
||||
) as EIDs on EIDs.id=IEIDs.equivalentid;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
s.idMatchStatement, err = s.db.Prepare(`
|
||||
select IDs.domain, IDs.stringid from IDs
|
||||
join IDsToEquivalantIDs as IEIDs on IDs.id=IEIDs.idid
|
||||
join (
|
||||
select EIDs.* from EquivalentIDs as EIDs
|
||||
join IDsToEquivalantIDs as QIEIDs on EIDs.id=QIEIDs.equivalentid
|
||||
join IDs as QIDs on QIDs.id=QIEIDs.idid
|
||||
where (QIDs.domain=? AND QIDs.stringid=?)
|
||||
) as EIDs on EIDs.id=IEIDs.equivalentid;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare database statements: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewSqliteStorage(db, path string) (HashStorage, error) {
|
||||
@ -411,34 +478,42 @@ func NewSqliteStorage(db, path string) (HashStorage, error) {
|
||||
}
|
||||
sqlite.db = sqlDB
|
||||
_, err = sqlite.db.Exec(`
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE IF NOT EXISTS Hashes(
|
||||
hashid INTEGER PRIMARY KEY,
|
||||
hash INTEGER NOT NULL,
|
||||
kind INTEGER NOT NULL,
|
||||
id INTEGER NOT NULL,
|
||||
FOREIGN KEY(id) REFERENCES IDs(idid),
|
||||
UNIQUE(kind, hash)
|
||||
);
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE IF NOT EXISTS IDs(
|
||||
id INTEGER PRIMARY KEY,
|
||||
stringid TEXT NOT NULL,
|
||||
domain TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS IDs(
|
||||
id TEXT NOT NULL,
|
||||
domain TEXT NOT NULL,
|
||||
idid INTEGER PRIMARY KEY,
|
||||
UNIQUE (domain, id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS Hashes(
|
||||
hash INTEGER NOT NULL,
|
||||
kind INTEGER NOT NULL,
|
||||
id INTEGER NOT NULL,
|
||||
|
||||
CREATE TABLE IF NOT EXISTS EquivalentIDs(
|
||||
id INTEGER
|
||||
groupid INTEGER,
|
||||
FOREIGN KEY(idid) REFERENCES IDs(idid)
|
||||
UNIQUE (groupid, id)
|
||||
);
|
||||
`)
|
||||
FOREIGN KEY(id) REFERENCES IDs(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS EquivalentIDs(
|
||||
id integer primary key
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS IDsToEquivalantIDs(
|
||||
idid INTEGER NOT NULL,
|
||||
equivalentid INTEGER NOT NULL,
|
||||
PRIMARY KEY (idid, equivalentid),
|
||||
|
||||
FOREIGN KEY(idid) REFERENCES IDs(id),
|
||||
FOREIGN KEY(equivalentid) REFERENCES EquivalentIDs(id)
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
sqlite.createIndexes()
|
||||
sqlite.db.SetMaxOpenConns(1)
|
||||
err = sqlite.PrepareStatements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sqlite, nil
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user