go-bindata实现redis lua脚本高效技巧

实际写业务的时候, 在使用redis时, 为了原子性, 经常会使用到嵌入lua脚本的方法, 把逻辑包装在lua文件中, 发到redis-server原子执行

下面以对消息队列的消息做幂等性判重的业务来介绍一种高效的方案

借助redis的set来实现判重, lua脚本(check_seq.lua)如下:

local seqKey        = KEYS[1]
local seqID            = ARGV[1]
local expireTime     = ARGV[2]

local ret = redis.call('sadd', seqKey, seqID)
if ret == 1 then
    redis.call('expire', seqKey, expireTime)
end

常见方案

一般开发者, 理所当然的迅速写下如下代码:

path := "res/lua/check_seq.lua"
f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
if err != nil {
    log.Fatal(err)
}
defer f.Close()

_, err = f.Read(data)
if err != nil {
    log.Fatal(err)
}

script := redis.NewScript(1, string(data))
r, err := redis.Int64(script.Do(client, KeyTimerLock, ip, expireTime))
if err != nil {
    log.Fatal(err)
}

此做法, 功能是没问题的. 但是仔细想想, 当业务体量大, 高并发时, 每次频繁的打开关闭文件, 会消耗大量不必要的资源, 从而影响服务的性能

当然, 有人会说, 服务启动时打开文件, 服务关闭时一起关闭所有lua文件, 这样就可以避免每次使用redis lua脚本时频繁打开关闭lua文件的性能消耗. 这样做有不好的地方: 1.忘记关闭文件, 造成资源泄露. 2.文件过多或多个服务在同一台机器时, 同时打开很多文件, 会造成资源的浪费

个人觉得比较好的方案是利用go的go-bindata打包静态资源的方法, 把lua脚本打包为静态资源文件, 从而可以直接读取lua脚本的二进制数据, 避免运行时文件的打开. 唯一的坏处就是打包的服务可执行文件较大

高效方案

通过go-bindata把 redis lua文件打包为go静态代码(会生成bindata.go文件, 里面都是静态资源), 避免每次redis.NewScript时因频繁打开lua文件, 影响性能

项目目录组织如下:

.
├── assets
│   ├── go-bindata
│   ├── go-bindata-osx
│   └── res
│       └── lua

go-bindata版本:

linux环境: go-bindata
win环境: go-bindata-osx

在项目的assets目录, 执行以下命令 ./go-bindata -pkg=assets -ignore=\.git res/…

.
├── assets
│   ├── bindata.go
│   ├── go-bindata
│   ├── go-bindata-osx
│   └── res
│       └── lua

可以看到, 生成了 bindata.go文件(文章末尾)

bindata.go的_resFts_myself_scriptLua就是check_seq.lua的二进制数据, 按照以下方法即可得到lua文件内容

path := "res/lua/check_seq.lua"
data, err = assets.Asset(path)
if err != nil {
    log.Fatal(err)
}

script := redis.NewScript(1, string(data))
r, err := redis.Int64(script.Do(client, KeyTimerLock, ip, expireTime))
if err != nil {
    log.Fatal(err)
}

assets.Asset()函数:

func Asset(name string) ([]byte, error) {
    cannonicalName := strings.Replace(name, "\\", "/", -1)
    if f, ok := _bindata[cannonicalName]; ok {
        a, err := f()
        if err != nil {
            return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
        }
        return a.bytes, nil
    }
    return nil, fmt.Errorf("Asset %s not found", name)
}

直接是根据传入的name从_bindata(map结构)中获取对应的二进制数据

var _bindata = map[string]func() (*asset, error){
    "res/check_seq.lua": resCheck_seqLua,
}

通过ggo-bindata, 生成静态资源的方式, 直接打入可执行文件, 并借助生成代码来直接获取lua文件内容, 高效读取lua文件二进制数据, 避免了使用redis的lua脚本方式的频繁打开关闭文件的性能损失

总结

善用go自带的工具, 高效解决问题

附件

bindata.go:

// Code generated by go-bindata.
// sources:
// res/check_seq.lua
// DO NOT EDIT!

package asset

import (
    "bytes"
    "compress/gzip"
    "fmt"
    "io"
    "io/ioutil"
    "os"
    "path/filepath"
    "strings"
    "time"
)

func bindataRead(data []byte, name string) ([]byte, error) {
    gz, err := gzip.NewReader(bytes.NewBuffer(data))
    if err != nil {
        return nil, fmt.Errorf("Read %q: %v", name, err)
    }

    var buf bytes.Buffer
    _, err = io.Copy(&buf, gz)
    clErr := gz.Close()

    if err != nil {
        return nil, fmt.Errorf("Read %q: %v", name, err)
    }
    if clErr != nil {
        return nil, err
    }

    return buf.Bytes(), nil
}

type asset struct {
    bytes []byte
    info  os.FileInfo
}

type bindataFileInfo struct {
    name    string
    size    int64
    mode    os.FileMode
    modTime time.Time
}

func (fi bindataFileInfo) Name() string {
    return fi.name
}
func (fi bindataFileInfo) Size() int64 {
    return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
    return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
    return fi.modTime
}
func (fi bindataFileInfo) IsDir() bool {
    return false
}
func (fi bindataFileInfo) Sys() interface{} {
    return nil
}

var _resCheck_seqLua = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xca\xc9\x4f\x4e\xcc\x51\x28\x4e\x2d\xf4\x4e\xad\xe4\xe4\xb4\x55\xf0\x76\x8d\x0c\x8e\x36\x8c\xe5\xe5\x82\x4b\x78\xba\x28\x00\x01\x48\xce\x31\xc8\x3d\x0c\x49\x2e\xb5\xa2\x20\xb3\x28\x35\x24\x33\x37\x55\x01\x26\x69\x04\x94\x84\x49\x17\xa5\x96\x28\xd8\x02\xc9\x94\xcc\x62\x3d\x20\x3f\x47\x43\xbd\x38\x31\x25\x45\x5d\x07\x6a\x99\x0e\xc4\x6c\x4d\x5e\xae\xcc\x34\x88\x5a\x5b\x05\x43\x85\x92\x8c\xd4\x3c\x5e\x2e\x90\x85\xc8\x1a\x21\x36\x21\x69\x45\x58\x0d\xd4\x9f\x9a\x97\x02\x08\x00\x00\xff\xff\xea\x11\xff\x1c\xc6\x00\x00\x00")


func resCheck_seqLuaBytes() ([]byte, error) {
    return bindataRead(
        _resCheck_seqLua,
        "res/check_seq.lua",
    )
}

func resCheck_seqLua() (*asset, error) {
    bytes, err := resCheck_seqLuaBytes()
    if err != nil {
        return nil, err
    }

    info := bindataFileInfo{name: "res/check_seq.lua", size: 198, mode: os.FileMode(436), modTime: time.Unix(1609840881, 0)}
    a := &asset{bytes: bytes, info: info}
    return a, nil

}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
    cannonicalName := strings.Replace(name, "\\", "/", -1)
    if f, ok := _bindata[cannonicalName]; ok {
        a, err := f()
        if err != nil {
            return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
        }
        return a.bytes, nil
    }
    return nil, fmt.Errorf("Asset %s not found", name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
    a, err := Asset(name)
    if err != nil {
        panic("asset: Asset(" + name + "): " + err.Error())
    }

    return a
}

// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
    cannonicalName := strings.Replace(name, "\\", "/", -1)
    if f, ok := _bindata[cannonicalName]; ok {
        a, err := f()
        if err != nil {
            return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
        }
        return a.info, nil
    }
    return nil, fmt.Errorf("AssetInfo %s not found", name)
}

// AssetNames returns the names of the assets.
func AssetNames() []string {
    names := make([]string, 0, len(_bindata))
    for name := range _bindata {
        names = append(names, name)
    }
    return names
}

// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
    "res/check_seq.lua": resCheck_seqLua,
}

// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
//     data/
//       foo.txt
//       img/
//         a.png
//         b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
    node := _bintree
    if len(name) != 0 {
        cannonicalName := strings.Replace(name, "\\", "/", -1)
        pathList := strings.Split(cannonicalName, "/")
        for _, p := range pathList {
            node = node.Children[p]
            if node == nil {
                return nil, fmt.Errorf("Asset %s not found", name)
            }
        }
    }
    if node.Func != nil {
        return nil, fmt.Errorf("Asset %s not found", name)
    }
    rv := make([]string, 0, len(node.Children))
    for childName := range node.Children {
        rv = append(rv, childName)
    }
    return rv, nil
}

type bintree struct {
    Func     func() (*asset, error)
    Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
    "res": &bintree{nil, map[string]*bintree{
        "check_seq.lua": &bintree{resCheck_seqLua, map[string]*bintree{}},
    }},
}}

// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
    data, err := Asset(name)
    if err != nil {
        return err
    }
    info, err := AssetInfo(name)
    if err != nil {
        return err
    }
    err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
    if err != nil {
        return err
    }
    err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
    if err != nil {
        return err
    }
    err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
    if err != nil {
        return err
    }
    return nil
}

// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
    children, err := AssetDir(name)
    // File
    if err != nil {
        return RestoreAsset(dir, name)
    }
    // Dir
    for _, child := range children {
        err = RestoreAssets(dir, filepath.Join(name, child))
        if err != nil {
            return err
        }
    }
    return nil
}

func _filePath(dir, name string) string {
    cannonicalName := strings.Replace(name, "\\", "/", -1)
    return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

go-bindata

redis lua脚本高效技巧

go-bindata实现redis lua脚本高效技巧

常见方案

高效方案

总结

附件

CATALOG

FEATURED TAGS