flygrounder/mtg-price-bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Redis -> YDB

Browse source
This commit is contained in:
Artyom Belousov 2023-05-27 22:10:31 +03:00
parent 1f4888069f
commit a33680d527
21 changed files with 448 additions and 322 deletions
Download patch file Download diff file Expand all files Collapse all files

100
internal/caching/cache.go
View file

@ -1,41 +1,107 @@
package caching
import (
"context"
"encoding/json"
"fmt"
"path"
"time"
"github.com/go-redis/redis"
"github.com/pkg/errors"
"github.com/ydb-platform/ydb-go-sdk/v3/table"
"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
"gitlab.com/flygrounder/go-mtg-vk/internal/cardsinfo"
)
type CacheClient struct {
Storage *redis.Client
Storage table.Client
Expiration time.Duration
Prefix string
}
func NewClient(addr string, passwd string, expiration time.Duration, db int) *CacheClient {
return &CacheClient{
Storage: redis.NewClient(&redis.Options{
Addr: addr,
Password: passwd,
DB: db,
}),
Expiration: expiration,
}
func (client *CacheClient) Init(ctx context.Context) error {
return client.Storage.Do(ctx, func(ctx context.Context, s table.Session) error {
return s.CreateTable(
ctx,
path.Join(client.Prefix, "cache"),
options.WithColumn("card", types.TypeString),
options.WithColumn("prices", types.Optional(types.TypeJSON)),
options.WithColumn("created_at", types.Optional(types.TypeTimestamp)),
options.WithTimeToLiveSettings(
options.NewTTLSettings().ColumnDateType("created_at").ExpireAfter(client.Expiration),
),
options.WithPrimaryKeyColumn("card"),
)
})
}
func (client *CacheClient) Set(key string, prices []cardsinfo.ScgCardPrice) {
func (client *CacheClient) Set(ctx context.Context, key string, prices []cardsinfo.ScgCardPrice) error {
const query = `
DECLARE $cacheData AS List<Struct<
card: String,
prices: Json,
created_at: Timestamp>>;
INSERT INTO cache SELECT cd.card AS card, cd.prices AS prices, cd.created_at AS created_at FROM AS_TABLE($cacheData) cd LEFT OUTER JOIN cache c ON cd.card = c.card WHERE c.card IS NULL`
value, _ := json.Marshal(prices)
client.Storage.Set(key, value, client.Expiration)
return client.Storage.Do(ctx, func(ctx context.Context, s table.Session) error {
_, _, err := s.Execute(ctx, writeTx(), query, table.NewQueryParameters(
table.ValueParam("$cacheData", types.ListValue(
types.StructValue(
types.StructFieldValue("card", types.StringValueFromString(key)),
types.StructFieldValue("prices", types.JSONValueFromBytes(value)),
types.StructFieldValue("created_at", types.TimestampValueFromTime(time.Now())),
))),
))
return err
})
}
func (client *CacheClient) Get(key string) ([]cardsinfo.ScgCardPrice, error) {
c, err := client.Storage.Get(key).Result()
func (client *CacheClient) Get(ctx context.Context, key string) ([]cardsinfo.ScgCardPrice, error) {
const query = `
DECLARE $card AS String;
SELECT UNWRAP(prices) AS prices FROM cache WHERE card = $card`
var pricesStr string
err := client.Storage.Do(ctx, func(ctx context.Context, s table.Session) error {
_, res, err := s.Execute(ctx, readTx(), query, table.NewQueryParameters(
table.ValueParam("$card", types.StringValueFromString(key)),
))
if err != nil {
return err
}
ok := res.NextResultSet(ctx)
if !ok {
return errors.New("no key")
}
ok = res.NextRow()
if !ok {
return errors.New("no key")
}
err = res.ScanNamed(
named.Required("prices", &pricesStr),
)
return err
})
if err != nil {
return nil, errors.Wrap(err, "No such key in cache")
fmt.Println(err.Error())
return nil, errors.Wrap(err, "Failed to get key")
}
var prices []cardsinfo.ScgCardPrice
json.Unmarshal([]byte(c), &prices)
json.Unmarshal([]byte(pricesStr), &prices)
return prices, nil
}
func writeTx() *table.TransactionControl {
return table.TxControl(table.BeginTx(
table.WithSerializableReadWrite(),
), table.CommitTx())
}
func readTx() *table.TransactionControl {
return table.TxControl(table.BeginTx(
table.WithOnlineReadOnly(),
), table.CommitTx())
}

64
internal/caching/cache_test.go
View file

@ -1,64 +0,0 @@
package caching
import (
"fmt"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/go-redis/redis"
"github.com/stretchr/testify/assert"
"gitlab.com/flygrounder/go-mtg-vk/internal/cardsinfo"
)
func TestGetClient(t *testing.T) {
c := NewClient("addr", "123", time.Hour, 1)
assert.Equal(t, time.Hour, c.Expiration)
assert.Equal(t, 1, c.Storage.Options().DB)
assert.Equal(t, "addr", c.Storage.Options().Addr)
assert.Equal(t, "123", c.Storage.Options().Password)
}
func TestGetSet(t *testing.T) {
client, s := getTestClient()
defer s.Close()
keyName := "test_key"
value := []cardsinfo.ScgCardPrice{
{
Price: "1",
Edition: "Alpha",
Link: "scg",
},
}
client.Set(keyName, value)
val, err := client.Get(keyName)
assert.Nil(t, err)
assert.Equal(t, value, val)
}
func TestExpiration(t *testing.T) {
client, s := getTestClient()
defer s.Close()
client.Expiration = time.Millisecond
keyName := "test_key"
var value []cardsinfo.ScgCardPrice
client.Set(keyName, value)
s.FastForward(time.Millisecond * 2)
val, err := client.Get(keyName)
assert.Nil(t, val)
assert.NotNil(t, err)
}
func getTestClient() (*CacheClient, *miniredis.Miniredis) {
s, _ := miniredis.Run()
fmt.Println(s.Addr())
c := redis.NewClient(&redis.Options{
Addr: s.Addr(),
})
return &CacheClient{
Storage: c,
Expiration: 0,
}, s
}

1
internal/cardsinfo/fetcher.go
View file

@ -1,5 +1,4 @@
package cardsinfo
type Fetcher struct {
Dict map[string]string
}

5
internal/cardsinfo/names.go
View file

@ -6,8 +6,6 @@ import (
"net/http"
"net/url"
"strings"
"gitlab.com/flygrounder/go-mtg-vk/internal/dicttranslate"
)
const scryfallUrl = "https://api.scryfall.com"
@ -23,9 +21,6 @@ func (f *Fetcher) GetNameByCardId(set string, number string) string {
func (f *Fetcher) GetOriginalName(name string) string {
path := scryfallUrl + "/cards/named?fuzzy=" + applyFilters(name)
result := getCardByUrl(path)
if result == "" && f.Dict != nil {
result, _ = dicttranslate.Find(name, f.Dict, 5)
}
return result
}

15
internal/cardsinfo/names_test.go
View file

@ -31,21 +31,6 @@ func TestGetOriginalName_Scryfall(t *testing.T) {
assert.Equal(t, "Result Card", name)
}
func TestGetOriginalName_DictTwice(t *testing.T) {
defer gock.Off()
gock.New(scryfallUrl + "/cards/named?fuzzy=card").Persist().Reply(http.StatusOK).JSON(card{})
f := &Fetcher{
Dict: map[string]string{
"card": "Card",
},
}
name := f.GetOriginalName("card")
assert.Equal(t, "Card", name)
name = f.GetOriginalName("card")
assert.Equal(t, "Card", name)
}
func TestGetOriginalName_BadJson(t *testing.T) {
defer gock.Off()

10
internal/dicttranslate/find.go
View file

@ -1,10 +0,0 @@
package dicttranslate
func Find(query string, dict map[string]string, maxDist int) (string, bool) {
var keys []string
for i := range dict {
keys = append(keys, i)
}
key, f := match(query, keys, maxDist)
return dict[key], f
}

21
internal/dicttranslate/find_test.go
View file

@ -1,21 +0,0 @@
package dicttranslate
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestFindEmpty(t *testing.T) {
dict := map[string]string{}
_, f := Find("", dict, 0)
assert.False(t, f)
}
func TestFindEntry(t *testing.T) {
dict := map[string]string{
"entry": "value",
}
val, f := Find("entry", dict, 0)
assert.True(t, f)
assert.Equal(t, "value", val)
}

24
internal/dicttranslate/match.go
View file

@ -1,24 +0,0 @@
package dicttranslate
import (
"github.com/texttheater/golang-levenshtein/levenshtein"
"strings"
)
func match(query string, opts []string, maxDist int) (string, bool) {
bestInd := -1
bestDist := 0
for i, s := range opts {
cfg := levenshtein.DefaultOptions
cfg.SubCost = 1
dist := levenshtein.DistanceForStrings([]rune(strings.ToLower(s)), []rune(strings.ToLower(query)), cfg)
if dist <= maxDist && (bestInd == -1 || dist < bestDist) {
bestInd = i
bestDist = dist
}
}
if bestInd == -1 {
return "", false
}
return opts[bestInd], true
}

59
internal/dicttranslate/match_test.go
View file

@ -1,59 +0,0 @@
package dicttranslate
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestMatch(t *testing.T) {
type testCase struct {
name string
query string
opts []string
shouldFind bool
match string
}
tests := []testCase{
{
name: "No options",
query: "opt",
opts: []string{},
shouldFind: false,
},
{
name: "Match one",
query: "option",
opts: []string{"opt1on"},
shouldFind: true,
match: "opt1on",
},
{
name: "Match exact",
query: "opt1on",
opts: []string{"option", "opt1on"},
shouldFind: true,
match: "opt1on",
},
{
name: "Do not match bad options",
query: "random",
opts: []string{"option", "opt1on"},
shouldFind: false,
},
{
name: "Match case insensitive",
query: "option",
opts: []string{"OPTION", "opt1on"},
shouldFind: true,
match: "OPTION",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
val, f := match(test.query, test.opts, 1)
assert.Equal(t, test.shouldFind, f)
assert.Equal(t, test.match, val)
})
}
}

16
internal/scenario/scenario.go
View file

@ -1,7 +1,9 @@
package scenario
import (
"context"
"errors"
"fmt"
"log"
"strings"
@ -27,8 +29,9 @@ type UserMessage struct {
}
type CardCache interface {
Get(cardName string) ([]cardsinfo.ScgCardPrice, error)
Set(cardName string, prices []cardsinfo.ScgCardPrice)
Init(ctx context.Context) error
Get(ctx context.Context, cardName string) ([]cardsinfo.ScgCardPrice, error)
Set(ctx context.Context, cardName string, prices []cardsinfo.ScgCardPrice) error
}
type CardInfoFetcher interface {
@ -42,7 +45,7 @@ type Sender interface {
SendPrices(userId int64, cardName string, prices []cardsinfo.ScgCardPrice)
}
func (s *Scenario) HandleSearch(msg *UserMessage) {
func (s *Scenario) HandleSearch(ctx context.Context, msg *UserMessage) {
cardName, err := s.getCardNameByCommand(msg.Body)
if err != nil {
s.Sender.Send(msg.UserId, incorrectMessage)
@ -51,7 +54,7 @@ func (s *Scenario) HandleSearch(msg *UserMessage) {
s.Sender.Send(msg.UserId, cardNotFoundMessage)
s.Logger.Printf("[info] Could not find card. User input: %s", msg.Body)
} else {
prices, err := s.Cache.Get(cardName)
prices, err := s.Cache.Get(ctx, cardName)
if err == nil {
s.Sender.SendPrices(msg.UserId, cardName, prices)
return
@ -62,7 +65,10 @@ func (s *Scenario) HandleSearch(msg *UserMessage) {
s.Logger.Printf("[error] Could not find SCG prices. Message: %s card name: %s", err.Error(), cardName)
return
}
s.Cache.Set(cardName, prices)
err = s.Cache.Set(ctx, cardName, prices)
if err != nil {
s.Logger.Println(fmt.Errorf("failed add entry in cache: %w", err))
}
s.Sender.SendPrices(msg.UserId, cardName, prices)
}
}

13
internal/scenario/scenario_test.go
View file

@ -1,6 +1,7 @@
package scenario
import (
"context"
"strings"
"testing"
@ -9,7 +10,7 @@ import (
func TestScenario_HandleSearch_BadCommand(t *testing.T) {
testCtx := GetTestScenarioCtx()
testCtx.Scenario.HandleSearch(&UserMessage{
testCtx.Scenario.HandleSearch(context.Background(), &UserMessage{
Body: "!s",
UserId: 1,
})
@ -24,7 +25,7 @@ func TestScenario_HandleSearch_BadCommand(t *testing.T) {
func TestScenario_HandleSearch_GoodCommand(t *testing.T) {
testCtx := GetTestScenarioCtx()
testCtx.Scenario.HandleSearch(&UserMessage{
testCtx.Scenario.HandleSearch(context.Background(), &UserMessage{
Body: "!s grn 228",
UserId: 1,
})
@ -38,7 +39,7 @@ func TestScenario_HandleSearch_GoodCommand(t *testing.T) {
func TestScenario_HandleSearch_NotFoundCard(t *testing.T) {
testCtx := GetTestScenarioCtx()
testCtx.Scenario.HandleSearch(&UserMessage{
testCtx.Scenario.HandleSearch(context.Background(), &UserMessage{
Body: "absolutely_random_card",
UserId: 1,
})
@ -53,7 +54,7 @@ func TestScenario_HandleSearch_NotFoundCard(t *testing.T) {
func TestScenario_HandleSearch_BadCard(t *testing.T) {
testCtx := GetTestScenarioCtx()
testCtx.Scenario.HandleSearch(&UserMessage{
testCtx.Scenario.HandleSearch(context.Background(), &UserMessage{
Body: "bad",
UserId: 1,
})
@ -67,7 +68,7 @@ func TestScenario_HandleSearch_BadCard(t *testing.T) {
}
func TestScenario_HandleSearch_Uncached(t *testing.T) {
testCtx := GetTestScenarioCtx()
testCtx.Scenario.HandleSearch(&UserMessage{
testCtx.Scenario.HandleSearch(context.Background(), &UserMessage{
Body: "uncached",
UserId: 1,
})
@ -77,6 +78,6 @@ func TestScenario_HandleSearch_Uncached(t *testing.T) {
message: "uncached",
},
}, testCtx.Sender.sent)
_, err := testCtx.Scenario.Cache.Get("uncached")
_, err := testCtx.Scenario.Cache.Get(context.Background(), "uncached")
assert.Nil(t, err)
}

10
internal/scenario/test_cache.go
View file

@ -1,6 +1,7 @@
package scenario
import (
"context"
"errors"
"gitlab.com/flygrounder/go-mtg-vk/internal/cardsinfo"
@ -10,7 +11,11 @@ type testCache struct {
table map[string][]cardsinfo.ScgCardPrice
}
func (t *testCache) Get(cardName string) ([]cardsinfo.ScgCardPrice, error) {
func (t *testCache) Init(ctx context.Context) error {
return nil
}
func (t *testCache) Get(ctx context.Context, cardName string) ([]cardsinfo.ScgCardPrice, error) {
msg, ok := t.table[cardName]
if !ok {
return nil, errors.New("test")
@ -18,6 +23,7 @@ func (t *testCache) Get(cardName string) ([]cardsinfo.ScgCardPrice, error) {
return msg, nil
}
func (t *testCache) Set(cardName string, prices []cardsinfo.ScgCardPrice) {
func (t *testCache) Set(ctx context.Context, cardName string, prices []cardsinfo.ScgCardPrice) error {
t.table[cardName] = prices
return nil
}

5
internal/vk/handler.go
View file

@ -1,6 +1,7 @@
package vk
import (
"context"
"net/http"
"github.com/gin-gonic/gin"
@ -8,7 +9,7 @@ import (
)
type Handler struct {
Scenario *scenario.Scenario
Scenario *scenario.Scenario
SecretKey string
GroupId int64
ConfirmationString string
@ -54,7 +55,7 @@ func (h *Handler) HandleMessage(c *gin.Context) {
case "confirmation":
h.handleConfirmation(c, &req)
case "message_new":
go h.Scenario.HandleSearch(&scenario.UserMessage{
go h.Scenario.HandleSearch(context.Background(), &scenario.UserMessage{
Body: req.Object.Body,
UserId: req.Object.UserId,
})