使用softmax进行动作选择？

对于某些机器学习应用程序，有一点需要将一组原始输出（例如来自神经网络的原始输出）映射到一组概率，并归一化为1。

在强化学习中，可能需要将一组可用操作的权重映射到一组相关的概率，然后将这些概率用于随机选择要采取的下一个操作。

Softmax函数通常用于将输出权重映射到一组相应的概率。“温度”参数允许对选择策略进行调整，在纯开采（“贪婪”策略，始终选择权重最高的动作）和纯探索（其中选择每个动作的可能性相等）之间进行插值。

这是使用Softmax函数的简单示例。每个“动作”对应vector<double>于此代码中传递的对象中的一个索引条目。

#include <iostream>
#include <iomanip>
#include <vector>
#include <random>
#include <cmath>


using std::vector;

// The temperature parameter here might be 1/temperature seen elsewhere.
// Here, lower temperatures move the highest-weighted output
// toward a probability of 1.0.
// And higer temperatures tend to even out all the probabilities,
// toward 1/<entry count>.
// temperature's range is between 0 and +Infinity (excluding these
// two extremes).
vector<double> Softmax(const vector<double>& weights, double temperature) {
    vector<double> probs;
    double sum = 0;
    for(auto weight : weights) {
        double pr = std::exp(weight/temperature);
        sum += pr;
        probs.push_back(pr);
    }
    for(auto& pr : probs) {
        pr /= sum;
    }
    return probs;
}

// Rng class encapsulates random number generation
// of double values uniformly distributed between 0 and 1,
// in case you need to replace std's <random> with something else.
struct Rng {
    std::mt19937 engine;
    std::uniform_real_distribution<double> distribution;
    Rng() : distribution(0,1) {
        std::random_device rd;
        engine.seed(rd());
    }
    double operator ()() {
        return distribution(engine);
    }
};

// Selects one index out of a vector of probabilities, "probs"
// The sum of all elements in "probs" must be 1.
vector<double>::size_type StochasticSelection(const vector<double>& probs) {

    // The unit interval is divided into sub-intervals, one for each
    // entry in "probs".  Each sub-interval's size is proportional
    // to its corresponding probability.

    // You can imagine a roulette wheel divided into differently-sized
    // slots for each entry.  An entry's slot size is proportional to
    // its probability and all the entries' slots combine to fill
    // the entire roulette wheel.

    // The roulette "ball"'s final location on the wheel is determined
    // by generating a (pseudo)random value between 0 and 1.
    // Then a linear search finds the entry whose sub-interval contains
    // this value.  Finally, the selected entry's index is returned.

    static Rng rng;
    const double point = rng();
    double cur_cutoff = 0;

    for(vector<double>::size_type i=0; i<probs.size()-1; ++i) {
        cur_cutoff += probs[i];
        if(point < cur_cutoff) return i;
    }
    return probs.size()-1;
}

void DumpSelections(const vector<double>& probs, int sample_count) {
    for(int i=0; i<sample_count; ++i) {
        auto selection = StochasticSelection(probs);
        std::cout << " " << selection;
    }
    std::cout << '\n';
}

void DumpDist(const vector<double>& probs) {
    auto flags = std::cout.flags();
    std::cout.precision(2);
    for(vector<double>::size_type i=0; i<probs.size(); ++i) {
        if(i) std::cout << "  ";
        std::cout << std::setw(2) << i << ':' << std::setw(8) << probs[i];
    }
    std::cout.flags(flags);
    std::cout << '\n';
}

int main() {
    vector<double> weights = {1.0, 2, 6, -2.5, 0};

    std::cout << "Original weights:\n";
    for(vector<double>::size_type i=0; i<weights.size(); ++i) {
        std::cout << "    " << i << ':' << weights[i];
    }
    std::cout << "\n\nSoftmax mappings for different temperatures:\n";
    auto softmax_thalf  = Softmax(weights, 0.5);
    auto softmax_t1     = Softmax(weights, 1);
    auto softmax_t2     = Softmax(weights, 2);
    auto softmax_t10    = Softmax(weights, 10);

    std::cout << "[Temp 1/2] ";
    DumpDist(softmax_thalf);
    std::cout << "[Temp 1]   ";
    DumpDist(softmax_t1);
    std::cout << "[Temp 2]   ";
    DumpDist(softmax_t2);
    std::cout << "[Temp 10]  ";
    DumpDist(softmax_t10);

    std::cout << "\nSelections from softmax_t1:\n";
    DumpSelections(softmax_t1, 20);
    std::cout << "\nSelections from softmax_t2:\n";
    DumpSelections(softmax_t2, 20);
    std::cout << "\nSelections from softmax_t10:\n";
    DumpSelections(softmax_t10, 20);
}

这是输出示例：

Original weights:
    0:1    1:2    2:6    3:-2.5    4:0

Softmax mappings for different temperatures:
[Temp 1/2]  0: 4.5e-05   1: 0.00034   2:       1   3: 4.1e-08   4: 6.1e-06
[Temp 1]    0:  0.0066   1:   0.018   2:    0.97   3:  0.0002   4:  0.0024
[Temp 2]    0:   0.064   1:    0.11   2:    0.78   3:   0.011   4:   0.039
[Temp 10]   0:    0.19   1:    0.21   2:    0.31   3:    0.13   4:    0.17

Selections from softmax_t1:
 2 2 2 2 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 1

Selections from softmax_t2:
 2 2 2 2 2 2 1 2 2 1 2 2 2 1 2 2 2 2 2 1

Selections from softmax_t10:
 0 0 4 1 2 2 2 0 0 1 3 4 2 2 4 3 2 1 0 1