diff --git a/resources/datetime.sas7bdat b/resources/datetime.sas7bdat new file mode 100644 index 00000000..a2e25c6f Binary files /dev/null and b/resources/datetime.sas7bdat differ diff --git a/src/sas/readstat_sas.h b/src/sas/readstat_sas.h index 5f4f2866..5d589107 100644 --- a/src/sas/readstat_sas.h +++ b/src/sas/readstat_sas.h @@ -111,8 +111,9 @@ typedef enum sas_subheader_type_e { #define SAS_PAGE_TYPE_AMD 0x0400 #define SAS_PAGE_TYPE_MASK 0x0F00 -#define SAS_PAGE_TYPE_META2 0x4000 -#define SAS_PAGE_TYPE_COMP 0x9000 +#define SAS_PAGE_TYPE_DELETED_ROWS 0x0080 +#define SAS_PAGE_TYPE_META2 0x4000 +#define SAS_PAGE_TYPE_COMP 0x9000 #define SAS_SUBHEADER_POINTER_SIZE_32BIT 12 #define SAS_SUBHEADER_POINTER_SIZE_64BIT 24 @@ -120,9 +121,10 @@ typedef enum sas_subheader_type_e { #define SAS_PAGE_HEADER_SIZE_32BIT 24 #define SAS_PAGE_HEADER_SIZE_64BIT 40 -#define SAS_COMPRESSION_NONE 0x00 -#define SAS_COMPRESSION_TRUNC 0x01 -#define SAS_COMPRESSION_ROW 0x04 +#define SAS_COMPRESSION_NONE 0x00 +#define SAS_COMPRESSION_TRUNC 0x01 +#define SAS_COMPRESSION_ROW 0x04 +#define SAS_COMPRESSION_DELETED_ROW 0x05 #define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL" #define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2" diff --git a/src/sas/readstat_sas7bdat_read.c b/src/sas/readstat_sas7bdat_read.c index 9d8107d2..5e32c1eb 100644 --- a/src/sas/readstat_sas7bdat_read.c +++ b/src/sas/readstat_sas7bdat_read.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "readstat_sas.h" #include "readstat_sas_rle.h" #include "../readstat_iconv.h" @@ -45,9 +46,12 @@ typedef struct sas7bdat_ctx_s { uint32_t row_length; uint32_t page_row_count; + uint32_t total_row_count; uint32_t parsed_row_count; + uint32_t parsed_deleted_row_count; uint32_t column_count; uint32_t row_limit; + uint32_t deleted_row_count; uint32_t row_offset; uint64_t header_size; @@ -232,7 +236,7 @@ static readstat_error_t sas7bdat_parse_column_size_subheader(const char *subhead static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader, size_t len, sas7bdat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; uint64_t total_row_count; - uint64_t row_length, page_row_count; + uint64_t row_length, deleted_row_count, page_row_count; if (len < (ctx->u64 ? 250: 190)) { retval = READSTAT_ERROR_PARSE; @@ -242,13 +246,22 @@ static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader, if (ctx->u64) { row_length = sas_read8(&subheader[40], ctx->bswap); total_row_count = sas_read8(&subheader[48], ctx->bswap); + deleted_row_count = sas_read8(&subheader[56], ctx->bswap); page_row_count = sas_read8(&subheader[120], ctx->bswap); } else { row_length = sas_read4(&subheader[20], ctx->bswap); total_row_count = sas_read4(&subheader[24], ctx->bswap); + deleted_row_count = sas_read4(&subheader[28], ctx->bswap); page_row_count = sas_read4(&subheader[60], ctx->bswap); } + if (deleted_row_count > total_row_count) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + ctx->total_row_count = total_row_count; + ctx->deleted_row_count = deleted_row_count; + sas_text_ref_t file_label_ref = sas7bdat_parse_text_ref(&subheader[len-130], ctx); if (file_label_ref.length) { if ((retval = sas7bdat_copy_text_ref(ctx->file_label, sizeof(ctx->file_label), @@ -275,7 +288,7 @@ static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader, } ctx->page_row_count = page_row_count; - uint64_t total_row_count_after_skipping = total_row_count; + uint64_t total_row_count_after_skipping = total_row_count - deleted_row_count; if (total_row_count > ctx->row_offset) { total_row_count_after_skipping -= ctx->row_offset; } else { @@ -393,6 +406,14 @@ static readstat_error_t sas7bdat_parse_column_format_subheader(const char *subhe return retval; } +static readstat_error_t sas7bdat_register_deleted_row(sas7bdat_ctx_t *ctx) { + if (ctx->parsed_deleted_row_count >= ctx->deleted_row_count) { + return READSTAT_ERROR_PARSE; + } + ctx->parsed_deleted_row_count++; + return READSTAT_OK; +} + static readstat_error_t sas7bdat_handle_data_value(readstat_variable_t *variable, col_info_t *col_info, const char *col_data, sas7bdat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; @@ -490,7 +511,15 @@ static readstat_error_t sas7bdat_parse_single_row(const char *data, sas7bdat_ctx return retval; } -static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len, sas7bdat_ctx_t *ctx) { +static unsigned char sas7bdat_read_bitmap(const unsigned char *bitmap, int index) { + unsigned char current_byte = bitmap[index / CHAR_BIT]; + unsigned char mask = 1 << (CHAR_BIT - 1 - index % CHAR_BIT); + + return current_byte & mask; +} + +static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len, + const unsigned char *deleted_bitmap, sas7bdat_ctx_t *ctx) { readstat_error_t retval = READSTAT_OK; int i; size_t row_offset=0; @@ -499,8 +528,13 @@ static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len, sas7bd retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH; goto cleanup; } - if ((retval = sas7bdat_parse_single_row(&data[row_offset], ctx)) != READSTAT_OK) + if (deleted_bitmap != NULL && sas7bdat_read_bitmap(deleted_bitmap, i)) { + if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if ((retval = sas7bdat_parse_single_row(&data[row_offset], ctx)) != READSTAT_OK) { goto cleanup; + } row_offset += ctx->row_length; } @@ -932,7 +966,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ goto cleanup; } } - } else if (shp_info.compression == SAS_COMPRESSION_ROW) { + } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; @@ -948,6 +982,25 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ return retval; } +static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, const char *data, + size_t page_size, const unsigned char **deleted_row_bitmap, sas7bdat_ctx_t *ctx) { + uint64_t page_unused_bytes; + if (ctx->u64) { + page_unused_bytes = sas_read8(&page[24], ctx->bswap); + } else { + page_unused_bytes = sas_read4(&page[12], ctx->bswap); + } + uint32_t row_count = ctx->page_row_count < ctx->total_row_count ? ctx->page_row_count : ctx->total_row_count; + uint64_t deleted_row_bitmap_offset = row_count * ctx->row_length + page_unused_bytes; + uint32_t required_bytes = row_count / CHAR_BIT + (row_count % CHAR_BIT == 0 ? 0 : 1); + + if ((data - page) + deleted_row_bitmap_offset + required_bytes > page_size) { + return READSTAT_ERROR_PARSE; + } + *deleted_row_bitmap = (const unsigned char *)data + deleted_row_bitmap_offset; + return READSTAT_OK; +} + static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { uint16_t page_type; @@ -1008,6 +1061,10 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) { goto cleanup; } + } else if (shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { + if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { + goto cleanup; + } } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; goto cleanup; @@ -1037,7 +1094,14 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ goto cleanup; } if (ctx->handle.value) { - retval = sas7bdat_parse_rows(data, page + page_size - data, ctx); + const unsigned char *deleted_row_bitmap = NULL; + if (page_type & SAS_PAGE_TYPE_DELETED_ROWS) { + if ((retval = sas7bdat_parse_deleted_row_bitmap(page, data, page_size, + &deleted_row_bitmap, ctx)) != READSTAT_OK) { + goto cleanup; + } + } + retval = sas7bdat_parse_rows(data, page + page_size - data, deleted_row_bitmap, ctx); } } cleanup: diff --git a/src/test/test_buffer.c b/src/test/test_buffer.c index 11c90413..1b869493 100644 --- a/src/test/test_buffer.c +++ b/src/test/test_buffer.c @@ -1,12 +1,20 @@ +#include #include +#include #include "test_buffer.h" rt_buffer_t *buffer_init(void) { rt_buffer_t *buffer = calloc(1, sizeof(rt_buffer_t)); + if (!buffer) { + return NULL; + } buffer->size = 1024; buffer->bytes = malloc(buffer->size); - + if (!buffer->bytes) { + free(buffer); + return NULL; + } return buffer; } @@ -36,3 +44,52 @@ void buffer_ctx_reset(rt_buffer_ctx_t *buffer_ctx) { buffer_reset(buffer_ctx->buffer); buffer_ctx->pos = 0; } + +readstat_error_t buffer_read_from_resource(rt_buffer_t *buffer, char *resource_name) { + const char *path_prefix = "resources/"; + int retval = READSTAT_OK; + + FILE* file = NULL; + char *resource_path = malloc(strlen(path_prefix) + strlen(resource_name) + 1); + if (resource_path == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + + strcpy(resource_path, path_prefix); + strcat(resource_path, resource_name); + file = fopen(resource_path, "rb"); + if (!file) { + retval = READSTAT_ERROR_OPEN; + goto cleanup; + } + + const size_t CHUNK_SIZE = 1024; + while (1) { + buffer_grow(buffer, CHUNK_SIZE); + if (!buffer->bytes) { + free(buffer); + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + + size_t bytes_read = fread(buffer->bytes + buffer->used, 1, CHUNK_SIZE, file); + buffer->used += bytes_read; + if (ferror(file)) { + buffer_free(buffer); + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + if (bytes_read < CHUNK_SIZE) { + break; + } + } + +cleanup: + free(resource_path); + if (file) { + fclose(file); + } + return retval; +} diff --git a/src/test/test_buffer.h b/src/test/test_buffer.h index cdceac05..9d36f786 100644 --- a/src/test/test_buffer.h +++ b/src/test/test_buffer.h @@ -1,3 +1,4 @@ +#include "../readstat.h" typedef struct rt_buffer_s { size_t used; @@ -17,3 +18,5 @@ void buffer_free(rt_buffer_t *buffer); rt_buffer_ctx_t *buffer_ctx_init(rt_buffer_t *buffer); void buffer_ctx_reset(rt_buffer_ctx_t *buffer_ctx); + +readstat_error_t buffer_read_from_resource(rt_buffer_t *buffer, char *resource_name); diff --git a/src/test/test_error.c b/src/test/test_error.c index 0f2a78f1..5cb213ad 100644 --- a/src/test/test_error.c +++ b/src/test/test_error.c @@ -181,7 +181,9 @@ static void print_value(readstat_value_t value) { void print_error(rt_error_t *error) { if (error->file) { - printf("Test \"%s\" failed: %s\n", error->file->label, error->msg); + rt_test_file_t *file = error->file; + char *test_name = file->label[0] ? file->label : file->resource_name; + printf("Test \"%s\" failed: %s\n", test_name, error->msg); } else { printf("Test failed: %s\n", error->msg); } diff --git a/src/test/test_list.h b/src/test/test_list.h index 35df0c18..3f5b7440 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -2317,6 +2317,66 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .label = "Resource tests", + .tests = { + { + .resource_name = "datetime.sas7bdat", + .table_name = "DATETIME", + .test_formats = RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE, + .rows = 3, + .columns_count = 5, + .columns = { + { + .name = "Date1", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "Date2", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } } + } + }, + { + .name = "DateTime", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 9538991236 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836 } } + } + }, + { + .name = "DateTimeHi", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836.85477447509766 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 9538991236.85477447509766 } } + } + }, + { + .name = "Taiw", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -17532 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -17532 } } + } + } + } + } + } } }; diff --git a/src/test/test_readstat.c b/src/test/test_readstat.c index 64f37ea6..a1bd9cc8 100644 --- a/src/test/test_readstat.c +++ b/src/test/test_readstat.c @@ -55,8 +55,11 @@ int main(int argc, char *argv[]) { int g, t, a, f; for (g=0; glabel[0] || file->resource_name[0])) + break; + for (a=0; aerrors_count; parse_ctx_reset(parse_ctx, f); - error = write_file_to_buffer(file, buffer, f); - if (error != file->write_error) { - push_error_if_codes_differ(parse_ctx, file->write_error, error); - error = READSTAT_OK; - continue; - } - if (error != READSTAT_OK) { - error = READSTAT_OK; - continue; + if (file->resource_name[0] == '\0') { + error = write_file_to_buffer(file, buffer, f); + if (error != file->write_error) { + push_error_if_codes_differ(parse_ctx, file->write_error, error); + error = READSTAT_OK; + continue; + } + if (error != READSTAT_OK) { + error = READSTAT_OK; + continue; + } + } else { + error = buffer_read_from_resource(buffer, file->resource_name); + if (error != READSTAT_OK) + break; } error = read_file(parse_ctx, f); @@ -108,9 +117,10 @@ int main(int argc, char *argv[]) { cleanup: if (error != READSTAT_OK) { dump_buffer(buffer, f); + rt_test_file_t *file = &_test_groups[g].tests[t]; + char *test_name = file->label[0] ? file->label : file->resource_name; printf("Error running test \"%s\" (format=%s): %s\n", - _test_groups[g].tests[t].label, - file_extension(f), readstat_error_message(error)); + test_name, file_extension(f), readstat_error_message(error)); buffer_free(buffer); return 1; } diff --git a/src/test/test_types.h b/src/test/test_types.h index f0a55e01..c731c6d8 100644 --- a/src/test/test_types.h +++ b/src/test/test_types.h @@ -65,6 +65,8 @@ typedef struct rt_test_file_s { long string_refs_count; char fweight[RT_MAX_STRING]; + + char resource_name[80]; } rt_test_file_t; typedef struct rt_test_group_s {