分词算法

逆向最大匹配

lua的中文处理是一个坑！太不方便了。

#!/usr/bin/env lua

-- Inverse maximum matching method

function contains(t, a)
    for i, b in ipairs(t) do
        if b == a then
            return true
        end
    end
    return false
end

dictionary = {'南京', '南京市', '南京市长', '市长', '长江', '长江大桥', '江大桥', '大桥', '桥'}

function imm(text, maxlen)
    -- inverse maximum matching method
    if maxlen == nil then
        maxlen=4
    end
    result = {}
    index = #text // 3
    while index > 0 do
        size = math.min(index, maxlen)
        while size > 0 do
            piece = string.sub(text, (index-size) * 3 + 1, index * 3)
            if contains(dictionary,  piece) then
                table.insert(result, 1, piece)
                index = index - (size - 1)
                break
            end
            size = size - 1
        end
        index = index - 1
    end
    return result
end

result = imm("南京市长江大桥", 4)
for _, a in ipairs(result) do
    print(a)
end