gensvm_consistency.c

Go to the documentation of this file.

#include "gensvm_consistency.h"

struct GenQueue *gensvm_top_queue(struct GenQueue *q, double percentile)
{
    long i, k, N = 0;
    double boundary,
           *perf = Calloc(double, q->N);
    struct GenQueue *nq = gensvm_init_queue();

    // find the desired percentile of performance
    for (i=0; i<q->N; i++) {
        perf[i] = q->tasks[i]->performance;
    }
    boundary = gensvm_percentile(perf, q->N, percentile);
    note("Boundary of the %g-th percentile determined at: %f\n",
            percentile, boundary);

    // find the number of tasks that perform at or above the boundary
    for (i=0; i<q->N; i++) {
        if (q->tasks[i]->performance >= boundary)
            N++;
    }

    // create a new queue with the best tasks
    nq->tasks = Malloc(struct GenTask *, N);
    k = 0;
    for (i=0; i<q->N; i++) {
        if (q->tasks[i]->performance >= boundary)
            nq->tasks[k++] = gensvm_copy_task(q->tasks[i]);
    }
    nq->N = N;
    nq->i = 0;

    free(perf);
    return nq;
}

void gensvm_consistency_repeats(struct GenQueue *q, long repeats,
        double percentile)
{
    bool breakout;
    long i, f, r, N, *cv_idx = NULL;
    double p, pi, pr, pt,
           *time = NULL,
           *std = NULL,
           *mean = NULL,
           *perf = NULL;
    struct GenQueue *nq = NULL;
    struct GenData **train_folds = NULL,
               **test_folds = NULL;
    struct GenModel *model = gensvm_init_model();
    struct GenTask *task = NULL;
    struct timespec loop_s, loop_e;

    nq = gensvm_top_queue(q, percentile);
    N = nq->N;

    note("Number of items to check: %li\n", nq->N);
    std = Calloc(double, N);
    mean = Calloc(double, N);
    time = Calloc(double, N);
    perf = Calloc(double, N*repeats);

    task = get_next_task(nq);

    model->n = 0;
    model->m = task->train_data->m;
    model->K = task->train_data->K;
    gensvm_allocate_model(model);
    gensvm_init_V(NULL, model, task->train_data);

    cv_idx = Calloc(long, task->train_data->n);

    train_folds = Malloc(struct GenData *, task->folds);
    test_folds = Malloc(struct GenData *, task->folds);

    i = 0;
    while (task) {
        gensvm_task_to_model(task, model);

        time[i] = 0.0;
        note("(%02li/%02li:%03li)\t", i+1, N, task->ID);
        for (r=0; r<repeats; r++) {
            Memset(cv_idx, long, task->train_data->n);
            gensvm_make_cv_split(task->train_data->n, task->folds, cv_idx);
            train_folds = Malloc(struct GenData *, task->folds);
            test_folds = Malloc(struct GenData *, task->folds);
            for (f=0; f<task->folds; f++) {
                train_folds[f] = gensvm_init_data();
                test_folds[f] = gensvm_init_data();
                gensvm_get_tt_split(task->train_data, train_folds[f],
                        test_folds[f], cv_idx, f);
                gensvm_kernel_preprocess(model, train_folds[f]);
                gensvm_kernel_postprocess(model, train_folds[f],
                        test_folds[f]);
            }

            Timer(loop_s);
            p = gensvm_cross_validation(model, train_folds, test_folds,
                    task->folds, task->train_data->n);
            Timer(loop_e);
            time[i] += gensvm_elapsed_time(&loop_s, &loop_e);
            matrix_set(perf, repeats, i, r, p);
            mean[i] += p/((double) repeats);
            note("%3.3f\t", p);
            // this is done because if we reuse the V it's not a
            // consistency check
            gensvm_init_V(NULL, model, task->train_data);
            for (f=0; f<task->folds; f++) {
                gensvm_free_data(train_folds[f]);
                gensvm_free_data(test_folds[f]);
            }
            free(train_folds);
            train_folds = NULL;

            free(test_folds);
            test_folds = NULL;
        }
        for (r=0; r<repeats; r++) {
            std[i] += pow(matrix_get(perf, repeats, i, r) - mean[i],
                    2.0);
        }
        if (r > 1) {
            std[i] /= ((double) repeats) - 1.0;
            std[i] = sqrt(std[i]);
        } else {
            std[i] = 0.0;
        }
        note("(m = %3.3f, s = %3.3f, t = %3.3f)\n", mean[i], std[i],
                time[i]);
        task = get_next_task(nq);
        i++;
    }

    // find the best overall configurations: those with high average
    // performance and low deviation in the performance
    note("\nBest overall configuration(s):\n");
    note("ID\tweights\tepsilon\t\tp\t\tkappa\t\tlambda\t\t"
            "mean_perf\tstd_perf\ttime_perf\n");
    p = 0.0;
    breakout = false;
    while (breakout == false) {
        pi = gensvm_percentile(mean, N, (100.0-p));
        pr = gensvm_percentile(std, N, p);
        pt = gensvm_percentile(time, N, p);
        for (i=0; i<N; i++)
            if ((pi - mean[i] < 0.0001) &&
                    (std[i] - pr < 0.0001) &&
                    (time[i] - pt < 0.0001)) {
                note("(%li)\tw = %li\te = %f\tp = %f\t"
                        "k = %f\tl = %f\t"
                        "mean: %3.3f\tstd: %3.3f\t"
                        "time: %3.3f\n",
                        nq->tasks[i]->ID,
                        nq->tasks[i]->weight_idx,
                        nq->tasks[i]->epsilon,
                        nq->tasks[i]->p,
                        nq->tasks[i]->kappa,
                        nq->tasks[i]->lambda,
                        mean[i],
                        std[i],
                        time[i]);
                breakout = true;
            }
        p += 1.0;
    }

    free(cv_idx);
    gensvm_free_model(model);
    gensvm_free_queue(nq);

    free(perf);
    free(std);
    free(mean);
    free(time);
}

int gensvm_dsort(const void *elem1, const void *elem2)
{
    const double t1 = (*(double *) elem1);
    const double t2 = (*(double *) elem2);
    return t1 > t2;
}

double gensvm_percentile(double *values, long N, double p)
{
    if (N == 1)
        return values[0];

    long i;
    double pi, pr, boundary;
    double *local = Malloc(double, N);
    for (i=0; i<N; i++)
        local[i] = values[i];

    qsort(local, N, sizeof(double), gensvm_dsort);
    p /= 100.0;
    p = p*N + 0.5;
    pi = maximum(minimum(floor(p), N-1), 1);
    pr = maximum(minimum(p - pi, 1), 0);
    boundary = (1 - pr)*local[((long) pi)-1] + pr*local[((long) pi)];

    free(local);

    return boundary;
}