大规模计算避免内存不足

我认为这将是一个相当困难甚至不可能的问题。我有一些代码以我的情景描述在这里为基础。

我将Python代码翻译成了Lua代码,但我发现了一个错误。

如果代码用于一个大小约为10的权重列表,它将正常工作,但在我的情况下,列表的权重高达60个,该代码将生成一个长度为n的所有可能的比特串数组,当n=60时,该数组长度将是2^60太大了。

有没有办法在生成如此大的数组之外计算它?我的最终代码将是在Lua中,但如果你知道其他任何语言的解决方案都可以,我以后仍然可以翻译它。

Python代码:

from math import prod
def bitstrings(n) :
    """Return all possible bitstrings of length n"""
    if n == 0 :
        yield []
        return
    else :
        for b in [0,1] :
            for x in bitstrings(n-1) :
                yield [b] + x

def prob_selected(weights, num_selected = 5) :

    # P(n generated, including e)*P(e of n selected | n generated including e)
    # i.e. Sum_n (n generated, including e) * #num_selections / #generated
    # num_selected = how many will be drawn out of the hat (at most)

    n = len(weights)
    final_probability = [0] * n

    for bits in bitstrings(n) :
        num_generated = sum(bits)
        prob_generated = prod([w if b else (1-w) for (w,b) in zip(weights, bits)])

        for i in range(n) :
            if bits[i] :
                final_probability[i] += prob_generated * min(num_selected, num_generated) / num_generated
    return final_probability

print(prob_selected([1, 1, 1, 1, 1,
                     0.5, 0.2, 0.8, 0.9, 0.1]))

Lua代码:

-- Python sum() 
table.reduce = function (list, fn)
    local acc
    for k, v in ipairs(list) do
        if 1 == k then
            acc = v
        else
            acc = fn(acc, v)
        end
    end
    return acc
end

local function generateBitstrings (global_arr, n, arr, i)
    if i == n then
        table.insert(global_arr, {table.unpack(arr)})
        return
    end

    arr[i] = 0
    generateBitstrings(global_arr, n, arr, i + 1)

    arr[i] = 1
    generateBitstrings(global_arr, n, arr, i + 1)
end

local function prob_selected (weights, num_selected)
    local n = #weights
    local final_probability = {}

    for i=1, n do
        final_probability[i] = 0
    end

    local globalArr = {}
    generateBitstrings(globalArr, n + 1, {}, 1)
    for ibots, bits in ipairs(globalArr) do
        local num_generated = table.reduce(
            bits,
            function(a, b)
                return a + b
            end
        )

        local prob_generated = 1
        local bitsLength = #bits
        for i=1,bitsLength do
            if bits[i] == 1 then
                prob_generated = prob_generated * weights[i]
            else
                prob_generated = prob_generated * (1 - weights[i])
            end
        end

        for i=1,n do
            if bits[i] == 1 then
                final_probability[i] = final_probability[i] + (prob_generated * math.min(num_selected, num_generated) / num_generated)
            end
        end
    end
    return final_probability
end

print (table.concat (prob_selected({1, 1, 1, 1, 1,0.5, 0.2, 0.8, 0.9, 0.1}, 5), ', '))
点赞