gensvm_io.c

Go to the documentation of this file.

#include "gensvm_io.h"

void gensvm_read_data(struct GenData *dataset, char *data_file)
{
    FILE *fid = NULL;
    long i, j, n, m,
         nr = 0,
         K = 0;
    double value;
    char buf[GENSVM_MAX_LINE_LENGTH];

    if ((fid = fopen(data_file, "r")) == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Datafile %s could not be opened.\n",
                data_file);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }

    // Read data dimensions
    nr += fscanf(fid, "%ld", &n);
    nr += fscanf(fid, "%ld", &m);

    // Allocate memory
    dataset->RAW = Malloc(double, n*(m+1));

    // Read first line of data
    for (j=1; j<m+1; j++) {
        nr += fscanf(fid, "%lf", &value);
        matrix_set(dataset->RAW, m+1, 0, j, value);
    }

    if (fgets(buf, GENSVM_MAX_LINE_LENGTH, fid) == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: No label found on first line.\n");
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }

    // Check if there is a label at the end of the line
    if (sscanf(buf, "%lf", &value) > 0) {
        dataset->y = Malloc(long, n);
        dataset->y[0] = value;
        K = 1;
    } else {
        free(dataset->y);
        dataset->y = NULL;
    }

    // Read the rest of the file
    for (i=1; i<n; i++) {
        for (j=1; j<m+1; j++) {
            nr += fscanf(fid, "%lf", &value);
            matrix_set(dataset->RAW, m+1, i, j, value);
        }
        if (dataset->y != NULL) {
            nr += fscanf(fid, "%lf", &value);
            dataset->y[i] = (long) value;
            K = maximum(K, dataset->y[i]);
        }
    }
    fclose(fid);

    if (nr < n * m) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: not enough data found in %s\n",
                data_file);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }

    // Set the column of ones
    for (i=0; i<n; i++)
        matrix_set(dataset->RAW, m+1, i, 0, 1.0);

    dataset->n = n;
    dataset->m = m;
    dataset->r = m;
    dataset->K = K;
    dataset->Z = dataset->RAW;

    if (gensvm_could_sparse(dataset->Z, n, m+1)) {
        note("Converting to sparse ... ");
        dataset->spZ = gensvm_dense_to_sparse(dataset->Z, n, m+1);
        note("done.\n");
        free(dataset->RAW);
        dataset->RAW = NULL;
        dataset->Z = NULL;
    }
}

void exit_input_error(int line_num)
{
    err("[GenSVM Error]: Wrong input format on line: %i\n", line_num);
    exit(EXIT_FAILURE);
}

void gensvm_read_data_libsvm(struct GenData *data, char *data_file)
{
    bool do_sparse, zero_based = false;
    long i, j, n, m, K, nnz, cnt, tmp, index, row_cnt, num_labels, 
         min_index = 1;
    int n_big, n_small, big_start;
    double value;
    FILE *fid = NULL;
    char *label = NULL,
         *endptr = NULL,
         **big_parts = NULL,
         **small_parts = NULL;
    char buf[GENSVM_MAX_LINE_LENGTH];

    fid = fopen(data_file, "r");
    if (fid == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Datafile %s could not be opened.\n",
                data_file);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }

    // first count the number of elements
    n = 0;
    m = -1;

    num_labels = 0;
    nnz = 0;

    while (fgets(buf, GENSVM_MAX_LINE_LENGTH, fid) != NULL) {
        // split the string in labels and/or index:value pairs
        big_parts = str_split(buf, " \t", &n_big);

        // record if this line has a label (first part has no colon)
        num_labels += (!str_contains_char(big_parts[0], ':'));

        // check for each part if it is a index:value pair
        for (i=0; i<n_big; i++) {
            if (!str_contains_char(big_parts[i], ':'))
                continue;

            // split the index:value pair
            small_parts = str_split(big_parts[i], ":", &n_small);

            // convert the index to a number
            index = strtol(small_parts[0], &endptr, 10);

            // catch conversion errors
            if (endptr == small_parts[0] || errno != 0 ||
                    *endptr != '\0')
                exit_input_error(n+1);

            // update the maximum index
            m = maximum(m, index);

            // update the minimum index
            min_index = minimum(min_index, index);

            // free the small parts
            for (j=0; j<n_small; j++) free(small_parts[j]);
            free(small_parts);

            // increment the nonzero counter
            nnz++;
        }

        // free the big parts
        for (i=0; i<n_big; i++) {
            free(big_parts[i]);
        }
        free(big_parts);

        // increment the number of observations
        n++;
    }

    // rewind the file pointer
    rewind(fid);

    // check if we have enough labels
    if (num_labels > 0 && num_labels != n) {
        err("[GenSVM Error]: There are some lines with missing "
                "labels. Please fix this before "
                "continuing.\n");
        exit(EXIT_FAILURE);
    }

    // don't forget the column of ones
    nnz += n;

    // deal with 0-based or 1-based indexing in the LibSVM file
    if (min_index == 0) {
        m++;
        zero_based = true;
    }

    // check if sparsity is worth it
    do_sparse = gensvm_nnz_comparison(nnz, n, m+1);
    if (do_sparse) {
        data->spZ = gensvm_init_sparse();
        data->spZ->nnz = nnz;
        data->spZ->n_row = n;
        data->spZ->n_col = m+1;
        data->spZ->values = Calloc(double, nnz);
        data->spZ->ia = Calloc(long, n+1);
        data->spZ->ja = Calloc(long, nnz);
        data->spZ->ia[0] = 0;
    } else {
        data->RAW = Calloc(double, n*(m+1));
        data->Z = data->RAW;
    }
    if (num_labels > 0)
        data->y = Calloc(long, n);

    K = 0;
    cnt = 0;
    for (i=0; i<n; i++) {
        fgets(buf, GENSVM_MAX_LINE_LENGTH, fid);

        // split the string in labels and/or index:value pairs
        big_parts = str_split(buf, " \t", &n_big);

        big_start = 0;
        // get the label from the first part if it exists
        if (!str_contains_char(big_parts[0], ':')) {
            label = strtok(big_parts[0], " \t\n");
            if (label == NULL) // empty line
                exit_input_error(i+1);

            // convert the label part to a number exit if there
            // are errors
            tmp = strtol(label, &endptr, 10);
            if (endptr == label || *endptr != '\0')
                exit_input_error(i+1);

            // assign label to y
            data->y[i] = tmp;

            // keep track of maximum K
            K = maximum(K, data->y[i]);

            // increment big part index
            big_start++;
        }

        row_cnt = 0;
        // set the first element in the row to 1
        if (do_sparse) {
            data->spZ->values[cnt] = 1.0;
            data->spZ->ja[cnt] = 0;
            cnt++;
            row_cnt++;
        } else {
            matrix_set(data->RAW, m+1, i, 0, 1.0);
        }

        // read the rest of the line
        for (j=big_start; j<n_big; j++) {
            if (!str_contains_char(big_parts[j], ':'))
                continue;

            // split the index:value pair
            small_parts = str_split(big_parts[j], ":", &n_small);
            if (n_small != 2)
                exit_input_error(n+1);

            // convert the index to a long
            errno = 0;
            index = strtol(small_parts[0], &endptr, 10);

            // catch conversion errors
            if (endptr == small_parts[0] || errno != 0 ||
                    *endptr != '\0')
                exit_input_error(n+1);

            // convert the value to a double
            errno = 0;
            value = strtod(small_parts[1], &endptr);
            if (endptr == small_parts[1] || errno != 0 ||
                    (*endptr != '\0' && !isspace(*endptr)))
                exit_input_error(n+1);

            if (do_sparse) {
                data->spZ->values[cnt] = value;
                data->spZ->ja[cnt] = index + zero_based;
                cnt++;
                row_cnt++;
            } else {
                matrix_set(data->RAW, m+1, i, 
                        index + zero_based, value);
            }

            // free the small parts
            free(small_parts[0]);
            free(small_parts[1]);
            free(small_parts);
        }

        if (do_sparse) {
            data->spZ->ia[i+1] = data->spZ->ia[i] + row_cnt;
        }

        // free the big parts
        for (j=0; j<n_big; j++) {
            free(big_parts[j]);
        }
        free(big_parts);
    }

    fclose(fid);

    data->n = n;
    data->m = m;
    data->r = m;
    data->K = K;

}

void gensvm_read_model(struct GenModel *model, char *model_filename)
{
    long i, j, nr = 0;
    FILE *fid = NULL;
    char buffer[GENSVM_MAX_LINE_LENGTH];
    char data_filename[GENSVM_MAX_LINE_LENGTH];
    double value = 0;

    fid = fopen(model_filename, "r");
    if (fid == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Couldn't open model file %s\n",
                model_filename);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }
    // skip the first four lines
    for (i=0; i<4; i++)
        next_line(fid, model_filename);

    // read all model variables
    model->p = get_fmt_double(fid, model_filename, "p = %lf");
    model->lambda = get_fmt_double(fid, model_filename, "lambda = %lf");
    model->kappa = get_fmt_double(fid, model_filename, "kappa = %lf");
    model->epsilon = get_fmt_double(fid, model_filename, "epsilon = %lf");
    model->weight_idx = (int) get_fmt_long(fid, model_filename,
            "weight_idx = %li");

    // skip to data section
    for (i=0; i<2; i++)
        next_line(fid, model_filename);

    // read filename of data file
    if (fgets(buffer, GENSVM_MAX_LINE_LENGTH, fid) == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Error reading from model file %s\n",
                model_filename);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }
    sscanf(buffer, "filename = %s\n", data_filename);
    model->data_file = Calloc(char, GENSVM_MAX_LINE_LENGTH);
    strcpy(model->data_file, data_filename);

    // read all data variables
    model->n = get_fmt_long(fid, model_filename, "n = %li\n");
    model->m = get_fmt_long(fid, model_filename, "m = %li\n");
    model->K = get_fmt_long(fid, model_filename, "K = %li\n");

    // skip to output
    for (i=0; i<2; i++)
        next_line(fid, model_filename);

    // read the matrix V and check for consistency
    model->V = Malloc(double, (model->m+1)*(model->K-1));
    for (i=0; i<model->m+1; i++) {
        for (j=0; j<model->K-1; j++) {
            nr += fscanf(fid, "%lf ", &value);
            matrix_set(model->V, model->K-1, i, j, value);
        }
    }
    if (nr != (model->m+1)*(model->K-1)) {
        // LCOV_EXCL_START
        err("[GenSVM Error] Error reading from model file %s. "
                "Not enough elements of V found.\n",
                model_filename);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }
}

void gensvm_write_model(struct GenModel *model, char *output_filename)
{
    FILE *fid = NULL;
    long i, j;
    char timestr[GENSVM_MAX_LINE_LENGTH];

    // open output file
    fid = fopen(output_filename, "w");
    if (fid == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Error opening output file %s\n",
                output_filename);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }
    gensvm_time_string(timestr);

    // Write output to file
    fprintf(fid, "Output file for GenSVM (version %s)\n", VERSION_STRING);
    fprintf(fid, "Generated on: %s\n\n", timestr);
    fprintf(fid, "Model:\n");
    fprintf(fid, "p = %15.16f\n", model->p);
    fprintf(fid, "lambda = %15.16f\n", model->lambda);
    fprintf(fid, "kappa = %15.16f\n", model->kappa);
    fprintf(fid, "epsilon = %g\n", model->epsilon);
    fprintf(fid, "weight_idx = %i\n", model->weight_idx);
    fprintf(fid, "\n");
    fprintf(fid, "Data:\n");
    fprintf(fid, "filename = %s\n", model->data_file);
    fprintf(fid, "n = %li\n", model->n);
    fprintf(fid, "m = %li\n", model->m);
    fprintf(fid, "K = %li\n", model->K);
    fprintf(fid, "\n");
    fprintf(fid, "Output:\n");
    for (i=0; i<model->m+1; i++) {
        for (j=0; j<model->K-1; j++) {
            if (j > 0)
                fprintf(fid, " ");
            fprintf(fid, "%+15.16f", matrix_get(model->V,
                        model->K-1, i, j));
        }
        fprintf(fid, "\n");
    }

    fclose(fid);
}

void gensvm_write_predictions(struct GenData *data, long *predy,
        char *output_filename)
{
    long i, j;
    FILE *fid = NULL;

    fid = fopen(output_filename, "w");
    if (fid == NULL) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Error opening output file %s\n",
                output_filename);
        exit(EXIT_FAILURE);
        // LCOV_EXCL_STOP
    }

    fprintf(fid, "%li\n", data->n);
    fprintf(fid, "%li\n", data->m);

    for (i=0; i<data->n; i++) {
        for (j=0; j<data->m; j++)
            fprintf(fid, "%.16f ", matrix_get(data->Z, data->m+1, i,
                        j+1));
        fprintf(fid, "%li\n", predy[i]);
    }

    fclose(fid);
}

void gensvm_time_string(char *buffer)
{
    int diff, hours, minutes;
    char timestr[GENSVM_MAX_LINE_LENGTH];
    time_t current_time, lt, gt;
    struct tm *lclt = NULL;

    // get current time (in epoch)
    current_time = time(NULL);
    if (current_time == ((time_t)-1)) {
        // LCOV_EXCL_START
        err("[GenSVM Error]: Failed to compute the current time.\n");
        return;
        // LCOV_EXCL_STOP
    }

    // convert time to local time and create a string
    lclt = localtime(&current_time);
    strftime(timestr, GENSVM_MAX_LINE_LENGTH, "%c", lclt);
    if (timestr == NULL) {
        err("[GenSVM Error]: Failed to convert time to string.\n");
        return;
    }

    // calculate the UTC offset including DST
    lt = mktime(localtime(&current_time));
    gt = mktime(gmtime(&current_time));
    diff = -difftime(gt, lt);
    hours = (diff/3600);
    minutes = (diff%3600)/60;
    if (lclt->tm_isdst == 1)
        hours++;

    sprintf(buffer, "%s (UTC %+03i:%02i)", timestr, hours, minutes);
}