diff --git a/cpp/src/file/read_file.cc b/cpp/src/file/read_file.cc index ce1f67197..c494de478 100644 --- a/cpp/src/file/read_file.cc +++ b/cpp/src/file/read_file.cc @@ -94,6 +94,7 @@ void ReadFile::close() { fd_ = -1; } file_size_ = -1; + file_version_ = 0; } int ReadFile::open(const std::string& file_path) { @@ -153,6 +154,20 @@ int ReadFile::check_file_magic() { 0) { ret = E_TSFILE_CORRUPTED; } + if (IS_FAIL(ret)) { + return ret; + } + + char version = 0; + if (RET_FAIL(read(MAGIC_STRING_TSFILE_LEN, &version, 1, read_len))) { + } else if (read_len != 1 || version != VERSION_NUM_BYTE) { + ret = E_TSFILE_CORRUPTED; + } else { + file_version_ = static_cast(version); + } + if (IS_FAIL(ret)) { + return ret; + } // file footer magic memset(buf, 0, MAGIC_STRING_TSFILE_LEN); diff --git a/cpp/src/file/read_file.h b/cpp/src/file/read_file.h index fbfe50a8b..9e3e958b7 100644 --- a/cpp/src/file/read_file.h +++ b/cpp/src/file/read_file.h @@ -31,7 +31,7 @@ namespace storage { class ReadFile { public: - ReadFile() : file_path_(), fd_(-1), file_size_(-1) {} + ReadFile() : file_path_(), fd_(-1), file_size_(-1), file_version_(0) {} ~ReadFile() { destroy(); } void destroy() { close(); } @@ -39,6 +39,7 @@ class ReadFile { FORCE_INLINE bool is_opened() const { return fd_ > 0; } FORCE_INLINE int64_t file_size() const { return file_size_; } FORCE_INLINE const std::string& file_path() const { return file_path_; } + FORCE_INLINE unsigned char file_version() const { return file_version_; } /** Return size and the Dataset Index v1 FNV fingerprint of size+mtime_ns. */ @@ -64,6 +65,7 @@ class ReadFile { std::string file_path_; int fd_; int64_t file_size_; + unsigned char file_version_; }; } // end namespace storage diff --git a/cpp/src/reader/filter/tag_filter.cc b/cpp/src/reader/filter/tag_filter.cc index 03b8ae785..94c0749c4 100644 --- a/cpp/src/reader/filter/tag_filter.cc +++ b/cpp/src/reader/filter/tag_filter.cc @@ -115,7 +115,7 @@ bool TagRegExp::satisfyRow(std::vector segments) const { !is_valid_pattern_) return false; try { - return std::regex_search(*segments[col_idx_], pattern_); + return std::regex_match(*segments[col_idx_], pattern_); } catch (const std::regex_error&) { return false; } @@ -137,7 +137,7 @@ bool TagNotRegExp::satisfyRow(std::vector segments) const { !is_valid_pattern_) return false; try { - return !std::regex_search(*segments[col_idx_], pattern_); + return !std::regex_match(*segments[col_idx_], pattern_); } catch (const std::regex_error&) { return true; } @@ -332,4 +332,4 @@ int TagFilterBuilder::get_tag_column_index(const std::string& columnName) { return idColumnOrder + 1; } -} // namespace storage \ No newline at end of file +} // namespace storage diff --git a/cpp/src/reader/tsfile_reader.cc b/cpp/src/reader/tsfile_reader.cc index 6e20b2d63..983a9ae95 100644 --- a/cpp/src/reader/tsfile_reader.cc +++ b/cpp/src/reader/tsfile_reader.cc @@ -127,6 +127,10 @@ int TsFileReader::open(const std::string& file_path) { return ret; } +unsigned char TsFileReader::get_file_version() const { + return read_file_ == nullptr ? 0 : read_file_->file_version(); +} + int TsFileReader::ensure_table_query_executor(int batch_size) { if (table_query_executor_ != nullptr && table_query_executor_batch_size_ == batch_size) { diff --git a/cpp/src/reader/tsfile_reader.h b/cpp/src/reader/tsfile_reader.h index 7b817a18c..f07ef3679 100644 --- a/cpp/src/reader/tsfile_reader.h +++ b/cpp/src/reader/tsfile_reader.h @@ -63,6 +63,7 @@ class TsFileReader { * @return Returns 0 on success, or a non-zero error code on failure. */ int close(); + unsigned char get_file_version() const; /** * @brief query the tsfile by the query expression,Users can construct * their own query expressions to query tsfile diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index f2d671998..3c3da441b 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -286,6 +286,8 @@ if (VENDORED_GTEST_INCLUDE_DIRS) endif () if (BUILD_TOOLS) target_include_directories(TsFile_Test PRIVATE ${CMAKE_SOURCE_DIR}/tools) + target_compile_definitions(TsFile_Test PRIVATE + TSFILE_CPP_SOURCE_DIR="${CMAKE_SOURCE_DIR}") endif () if (APPLE AND NOT MSVC) target_compile_options(TsFile_Test PRIVATE -std=c++14) diff --git a/cpp/test/reader/filter/tag_filter_test.cc b/cpp/test/reader/filter/tag_filter_test.cc index 02ce64b85..0a218b7a4 100644 --- a/cpp/test/reader/filter/tag_filter_test.cc +++ b/cpp/test/reader/filter/tag_filter_test.cc @@ -406,9 +406,13 @@ TEST_F(TagFilterTest, TagRegExpEdgeCases) { invalid_filter->satisfyRow(0, segments)); // handles gracefully cleanupSegments(segments); - // Empty pattern matches everything + // Full-value matching means an empty pattern only matches an empty value. auto empty_filter = builder_->reg_exp("name", ""); segments = createSegments("any", "25", "engineering", "active", "95"); + EXPECT_FALSE(empty_filter->satisfyRow(0, segments)); + cleanupSegments(segments); + + segments = createSegments("", "25", "engineering", "active", "95"); EXPECT_TRUE(empty_filter->satisfyRow(0, segments)); cleanupSegments(segments); @@ -497,4 +501,4 @@ TEST_F(TagFilterTest, TagIsNotNullFilter) { delete filter; delete trailing; -} \ No newline at end of file +} diff --git a/cpp/test/tools/cli_args_test.cc b/cpp/test/tools/cli_args_test.cc index 42b7eb650..bbb6e050f 100644 --- a/cpp/test/tools/cli_args_test.cc +++ b/cpp/test/tools/cli_args_test.cc @@ -34,6 +34,34 @@ TEST(RunCliTest, VersionFlagPrintsVersionAndReturnsOk) { EXPECT_TRUE(err.str().empty()); } +TEST(RunCliTest, VersionMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "--version", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--version"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, TopLevelHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help", "cat"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + +TEST(RunCliTest, CommandHelpMustAppearByItself) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "--help", "data.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); +} + TEST(RunCliTest, NoArgsPrintsUsageToErrAndReturnsUsageError) { std::ostringstream out; std::ostringstream err; @@ -54,7 +82,7 @@ TEST(RunCliTest, LeadingOptionBeforeCommandIsClearError) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"-f", "json", "meta", "data.tsfile"}, out, err); + tsfile_cli::run_cli({"-f", "ndjson", "meta", "data.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("command must come before options"), std::string::npos) @@ -69,14 +97,15 @@ TEST(ParseArgsTest, CommandAndFilePositional) { } TEST(ParseArgsTest, FormatFlagParsed) { - auto p = tsfile_cli::parse_args({"cat", "-f", "json", "data.tsfile"}); + auto p = tsfile_cli::parse_args({"cat", "-f", "ndjson", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.format, tsfile_cli::ParsedArgs::Format::kJson); } -TEST(ParseArgsTest, MeasurementsSplitOnComma) { - auto p = tsfile_cli::parse_args({"cat", "-m", "s1,s2,s3", "data.tsfile"}); - ASSERT_EQ(p.measurements.size(), 3u); +TEST(ParseArgsTest, MeasurementsRepeatOneNamePerOption) { + auto p = + tsfile_cli::parse_args({"cat", "-m", "s1", "-m", "s2", "data.tsfile"}); + ASSERT_EQ(p.measurements.size(), 2u); EXPECT_EQ(p.measurements[1], "s2"); } @@ -97,27 +126,24 @@ TEST(ParseArgsTest, TagFilterParsed) { {"cat", "--tag-filter", "id1", "eq", "dev_a", "data.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); + ASSERT_EQ(p.tag_filters.size(), 1u); + EXPECT_EQ(p.tag_filters[0].op, tsfile_cli::ParsedArgs::TagFilterOp::kEq); + EXPECT_EQ(p.tag_filters[0].column, "id1"); + EXPECT_EQ(p.tag_filters[0].value, "dev_a"); } -TEST(ParseArgsTest, TagBetweenParsed) { +TEST(ParseArgsTest, TagBetweenIsNotSupported) { auto p = tsfile_cli::parse_args( {"cat", "--tag-between", "id1", "dev_a", "dev_c", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_TRUE(p.has_tag_filter); - EXPECT_EQ(p.tag_filter_op, tsfile_cli::ParsedArgs::TagFilterOp::kBetween); - EXPECT_EQ(p.tag_filter_column, "id1"); - EXPECT_EQ(p.tag_filter_value, "dev_a"); - EXPECT_EQ(p.tag_filter_value2, "dev_c"); + EXPECT_FALSE(p.error.empty()); } TEST(ParseArgsTest, DuplicateTagFilterIsError) { auto p = tsfile_cli::parse_args({"cat", "--tag-filter", "id1", "eq", - "dev_a", "--tag-between", "id1", "a", "z", + "dev_a", "--tag-filter", "id1", "neq", "z", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); + EXPECT_TRUE(p.error.empty()); + ASSERT_EQ(p.tag_filters.size(), 2u); } TEST(ParseArgsTest, UnknownFlagIsError) { @@ -138,9 +164,9 @@ TEST(ParseArgsTest, MissingFileIsAllowedAtParseTime) { } TEST(ParseArgsTest, WriteFlagsParsed) { - auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-v", - "--header-match", "in.csv"}); + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", "in.csv", "-o", + "out.tsfile", "-v", "--header-match"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.command, "write"); EXPECT_EQ(p.table, "t1"); @@ -149,6 +175,7 @@ TEST(ParseArgsTest, WriteFlagsParsed) { EXPECT_TRUE(p.verbose); EXPECT_TRUE(p.header_match); EXPECT_EQ(p.file, "in.csv"); + EXPECT_TRUE(p.input_set); } TEST(ParseArgsTest, OutputFlagNeedsValue) { @@ -156,29 +183,12 @@ TEST(ParseArgsTest, OutputFlagNeedsValue) { EXPECT_FALSE(p.error.empty()); } -TEST(ParseArgsTest, DashIsStdinPositional) { - auto p = - tsfile_cli::parse_args({"write", "--table", "t1", "--columns", - "s1:INT64:field", "-o", "out.tsfile", "-"}); +TEST(ParseArgsTest, StdinFlagParsed) { + auto p = tsfile_cli::parse_args({"write", "--table", "t1", "--field", "s1", + "INT64", "--stdin", "-o", "out.tsfile"}); EXPECT_TRUE(p.error.empty()); EXPECT_EQ(p.file, "-"); -} - -TEST(ParseArgsTest, SeedFlagParsed) { - auto p = tsfile_cli::parse_args( - {"sample", "-m", "s1", "-n", "3", "--seed", "42", "data.tsfile"}); - EXPECT_TRUE(p.error.empty()); - EXPECT_EQ(p.command, "sample"); - EXPECT_EQ(p.limit, 3); - EXPECT_TRUE(p.has_seed); - EXPECT_EQ(p.seed, 42); -} - -TEST(ParseArgsTest, BadSeedValueIsError) { - auto p = tsfile_cli::parse_args( - {"sample", "--seed", "not_a_number", "data.tsfile"}); - EXPECT_FALSE(p.error.empty()); - EXPECT_NE(p.error.find("Invalid --seed"), std::string::npos); + EXPECT_TRUE(p.input_set); } TEST(RunCliTest, SelectIsNoLongerKnownCommand) { @@ -195,16 +205,14 @@ TEST(RunCliTest, SeedOnCatIsUsageError) { int code = tsfile_cli::run_cli({"cat", "--seed", "7", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--seed is only valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("--seed is not supported"), std::string::npos); } -TEST(RunCliTest, OffsetOnSampleIsUsageError) { +TEST(RunCliTest, SampleIsUnknownCommand) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"sample", "--offset", "2", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--offset is not valid for sample"), - std::string::npos); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos); } diff --git a/cpp/test/tools/cli_requirements_test.cc b/cpp/test/tools/cli_requirements_test.cc new file mode 100644 index 000000000..adfb1c4f6 --- /dev/null +++ b/cpp/test/tools/cli_requirements_test.cc @@ -0,0 +1,977 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#endif + +#include "cli/run_cli.h" +#include "cli_test_util.h" +#include "reader/tsfile_reader.h" + +#ifndef TSFILE_CPP_SOURCE_DIR +#define TSFILE_CPP_SOURCE_DIR "." +#endif + +namespace { + +struct TableFixture { + std::string path = tsfile_cli_test::write_table_fixture(); + ~TableFixture() { std::remove(path.c_str()); } +}; + +struct MultiTableFixture { + std::string path = tsfile_cli_test::write_multi_table_fixture(); + ~MultiTableFixture() { std::remove(path.c_str()); } +}; + +bool file_exists(const std::string& path) { + std::ifstream in(path.c_str()); + return in.good(); +} + +std::string read_file(const std::string& path) { + std::ifstream in(path.c_str(), std::ios::binary); + std::ostringstream buf; + buf << in.rdbuf(); + return buf.str(); +} + +} // namespace + +TEST(CliRequirements, HelpListsExactlyCurrentCommandSurface) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--help"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_NE(out.str().find( + "ls schema meta stats count sketch head cat export write"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("sample"), std::string::npos) << out.str(); + EXPECT_TRUE(err.str().empty()); +} + +TEST(CliRequirements, CommandHelpIsSpecificAndDocumentsSyntaxFieldsExamples) { + const std::vector commands = { + "ls", "schema", "meta", "stats", "count", + "sketch", "head", "cat", "export", "write"}; + for (const auto& command : commands) { + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({command, "--help"}, out, err), 0) + << command; + EXPECT_TRUE(err.str().empty()) << command << ": " << err.str(); + EXPECT_NE(out.str().find("Usage: tsfile-cli " + command), + std::string::npos) + << command << ": " << out.str(); + EXPECT_NE(out.str().find("Result fields:"), std::string::npos) + << command << ": " << out.str(); + EXPECT_NE(out.str().find("Examples:"), std::string::npos) + << command << ": " << out.str(); + EXPECT_EQ(out.str().find("Commands:"), std::string::npos) + << command << ": " << out.str(); + } + + std::ostringstream meta_out; + std::ostringstream meta_err; + EXPECT_EQ(tsfile_cli::run_cli({"meta", "--help"}, meta_out, meta_err), 0); + EXPECT_TRUE(meta_err.str().empty()); + EXPECT_NE(meta_out.str().find("Usage: tsfile-cli meta"), std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("Result fields:"), std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("size_bytes,format_version,model"), + std::string::npos) + << meta_out.str(); + EXPECT_NE(meta_out.str().find("Examples:"), std::string::npos) + << meta_out.str(); + EXPECT_EQ(meta_out.str().find("Commands:"), std::string::npos) + << meta_out.str(); + + std::ostringstream write_out; + std::ostringstream write_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "-h"}, write_out, write_err), 0); + EXPECT_TRUE(write_err.str().empty()); + EXPECT_NE(write_out.str().find("Usage: tsfile-cli write"), + std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("--field "), std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("Default: success is silent"), + std::string::npos) + << write_out.str(); + EXPECT_NE(write_out.str().find("Examples:"), std::string::npos) + << write_out.str(); + EXPECT_EQ(write_out.str().find("Commands:"), std::string::npos) + << write_out.str(); +} + +TEST(CliRequirements, SkillShipsRequiredReferenceFiles) { + const std::string root = std::string(TSFILE_CPP_SOURCE_DIR) + + "/tools/skills/tsfile-cli/references/"; + const std::vector refs = {"commands.md", "errors.md", + "examples.md"}; + for (const auto& ref : refs) { + std::ifstream in(root + ref); + ASSERT_TRUE(in.good()) << ref; + std::ostringstream body; + body << in.rdbuf(); + EXPECT_NE(body.str().find("tsfile-cli"), std::string::npos) + << root + ref; + } +} + +TEST(CliRequirements, SampleIsNotACommand) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sample", "x.tsfile"}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown command"), std::string::npos) + << err.str(); +} + +TEST(CliRequirements, VersionIncludesConcreteBuildMetadata) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"--version"}, out, err); + EXPECT_EQ(code, 0); + EXPECT_TRUE(err.str().empty()); + EXPECT_NE(out.str().find("tsfile-cli "), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" tsfile="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" commit="), std::string::npos) << out.str(); + EXPECT_NE(out.str().find(" built="), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("unknown"), std::string::npos) << out.str(); +} + +TEST(CliRequirements, FormatVocabularyIsTableNdjsonCsvOnly) { + TableFixture f; + + std::ostringstream ndjson_out; + std::ostringstream ndjson_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, + ndjson_out, ndjson_err), + 0) + << ndjson_err.str(); + EXPECT_EQ(ndjson_out.str(), + "{\"time\":\"0\",\"id1\":\"id1_field_1\",\"id2\":" + "\"id2_field_2\",\"s1\":\"0\"}\n"); + + std::ostringstream json_out; + std::ostringstream json_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "json", f.path}, json_out, json_err), + 1); + EXPECT_TRUE(json_out.str().empty()); + + std::ostringstream tsv_out; + std::ostringstream tsv_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "tsv", f.path}, tsv_out, tsv_err), 1); + EXPECT_TRUE(tsv_out.str().empty()); +} + +TEST(CliRequirements, DuplicateSingletonOptionsAreUsageErrors) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"cat", "-f", "csv", "-f", "ndjson", f.path}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--format specified more than once"), + std::string::npos) + << err.str(); + + const std::vector> duplicate_singletons = { + {"cat", "-n", "1", "--limit", "2", f.path}, + {"cat", "--offset", "0", "--offset", "1", f.path}, + {"cat", "--start", "0", "--start", "1", f.path}, + {"cat", "--end", "1", "--end", "2", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--output", + "b.csv", f.path}, + {"export", "-t", "table1", "--type", "csv", "--output-dir", "a", + "--output-dir", "b", f.path}, + {"export", "-t", "table1", "--type", "csv", "-o", "a.csv", "--force", + "--force", f.path}, + }; + for (const std::vector& args : duplicate_singletons) { + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli(args, dup_out, dup_err), 1) + << dup_err.str(); + EXPECT_TRUE(dup_out.str().empty()); + EXPECT_NE(dup_err.str().find("specified more than once"), + std::string::npos) + << dup_err.str(); + } +} + +TEST(CliRequirements, PositionalFileMustBeFinalUnlessAfterDoubleDash) { + TableFixture f; + + std::ostringstream bad_out; + std::ostringstream bad_err; + int bad_code = + tsfile_cli::run_cli({"cat", f.path, "-f", "csv"}, bad_out, bad_err); + EXPECT_EQ(bad_code, 1); + EXPECT_TRUE(bad_out.str().empty()); + EXPECT_NE(bad_err.str().find("Unexpected argument after file"), + std::string::npos) + << bad_err.str(); + + std::ostringstream ok_out; + std::ostringstream ok_err; + int ok_code = + tsfile_cli::run_cli({"cat", "-m", "s1", "--", f.path}, ok_out, ok_err); + EXPECT_EQ(ok_code, 0) << ok_err.str(); + EXPECT_TRUE(ok_err.str().empty()); +} + +TEST(CliRequirements, MeasurementOptionRepeatsAndRejectsCommaLists) { + TableFixture f; + std::ostringstream comma_out; + std::ostringstream comma_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1,s2", f.path}, comma_out, + comma_err), + 1); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-m", "s1", f.path}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); +} + +TEST(CliRequirements, MetaOnlyReturnsSizeFormatVersionAndModel) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(err.str().empty()); + EXPECT_EQ(out.str().substr( + 0, std::string("size_bytes,format_version,model\n").size()), + "size_bytes,format_version,model\n") + << out.str(); + EXPECT_EQ(out.str().find("path"), std::string::npos) << out.str(); + EXPECT_EQ(out.str().find("device_count"), std::string::npos) << out.str(); +} + +TEST(CliRequirements, LsReturnsModelAndObjectFields) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); +} + +TEST(CliRequirements, SchemaReturnsFixedSevenFieldContract) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str().substr( + 0, std::string("model,object,column,category,data_type," + "encoding,compression\n") + .size()), + "model,object,column,category,data_type,encoding,compression\n") + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,STRING"), std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos) + << out.str(); +} + +TEST(CliRequirements, HeadCatAndExportRejectOffsetWithZeroLimit) { + TableFixture f; + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_zero_limit_export", ".csv"); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-n", "0", "--offset", "1", f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-n", "0", "--offset", "1", f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + + std::ostringstream export_out; + std::ostringstream export_err; + EXPECT_EQ( + tsfile_cli::run_cli({"export", "-t", "table1", "--type", "csv", "-o", + out_path, "-n", "0", "--offset", "1", f.path}, + export_out, export_err), + 1); + EXPECT_TRUE(export_out.str().empty()); + EXPECT_FALSE(file_exists(out_path)); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, HeadAndCatRequireScopeForMultiObjectFiles) { + MultiTableFixture f; + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), + 1); + EXPECT_TRUE(cat_out.str().empty()); + EXPECT_NE(cat_err.str().find("requires -t/--table"), std::string::npos) + << cat_err.str(); + + std::ostringstream head_out; + std::ostringstream head_err; + EXPECT_EQ(tsfile_cli::run_cli({"head", "-m", "s1", "-f", "csv", f.path}, + head_out, head_err), + 1); + EXPECT_TRUE(head_out.str().empty()); + EXPECT_NE(head_err.str().find("requires -t/--table"), std::string::npos) + << head_err.str(); +} + +TEST(CliRequirements, WriteUsesExplicitTagAndFieldOptions) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_req_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,id1,s1\n0,dev,0\n1,dev,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_req_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + EXPECT_TRUE(file_exists(out_path)); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, + rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), "time,id1,s1\n0,dev,0\n1,dev,10\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteMapsInputByHeaderName) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_header_order", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "temp,time,site\n21.5,1000,beijing\n22.0,2000,shanghai\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_header_order_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", "--field", + "temp", "FLOAT", "-i", csv, "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream rout; + std::ostringstream rerr; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "sensors", "-f", "csv", out_path}, rout, rerr), + 0) + << rerr.str(); + EXPECT_EQ(rout.str(), + "time,site,temp\n1000,beijing,21.5\n2000,shanghai,22\n"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsHeaderShapeErrorsBeforeCreatingOutput) { + struct Case { + const char* name; + const char* content; + const char* needle; + }; + const Case cases[] = { + {"missing", "time,site\n1000,beijing\n", "missing required column"}, + {"undeclared", "time,site,temp,status\n1000,beijing,21.5,true\n", + "undeclared column"}, + {"duplicate", "time,site,temp,TEMP\n1000,beijing,21.5,22.0\n", + "conflict"}, + }; + for (const Case& c : cases) { + std::string csv = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_") + c.name, ".csv"); + { + std::ofstream o(csv.c_str()); + o << c.content; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + std::string("tsfile_cli_header_out_") + c.name, ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2) << c.name << " " << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find(c.needle), std::string::npos) + << c.name << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + +TEST(CliRequirements, WriteAppliesAndValidatesTypePhysicalOverrides) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_type_override", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp,humidity\n" + "1000,beijing,21.0,40.5\n" + "2000,beijing,21.5,41.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_type_override_out", ".tsfile"); + + std::ostringstream wout; + std::ostringstream werr; + int wc = tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", + "site", "STRING", "--field", "temp", + "FLOAT", "--field", "humidity", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--compression", + "FLOAT", "UNCOMPRESSED", "-i", csv, + "-o", out_path}, + wout, werr); + EXPECT_EQ(wc, 0) << werr.str(); + + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ( + tsfile_cli::run_cli({"schema", "-t", "sensors", "-f", "csv", out_path}, + schema_out, schema_err), + 0) + << schema_err.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,temp,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + EXPECT_NE(schema_out.str().find( + "table,sensors,humidity,FIELD,FLOAT,GORILLA,UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + + std::ostringstream dup_out; + std::ostringstream dup_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--field", "temp", "FLOAT", + "--encoding", "FLOAT", "GORILLA", "--encoding", "FLOAT", + "PLAIN", "--stdin", "-o", "unused.tsfile"}, + dup_out, dup_err), + 1); + EXPECT_NE(dup_err.str().find("specified more than once"), std::string::npos) + << dup_err.str(); + + std::ostringstream unused_out; + std::ostringstream unused_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "sensors", "--field", + "temp", "FLOAT", "--encoding", "DOUBLE", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + unused_out, unused_err), + 1); + EXPECT_NE(unused_err.str().find("not used"), std::string::npos) + << unused_err.str(); + + std::ostringstream bad_out; + std::ostringstream bad_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "binary_data", "--field", + "payload", "BLOB", "--encoding", "BLOB", + "GORILLA", "--stdin", "-o", "unused.tsfile"}, + bad_out, bad_err), + 1); + EXPECT_NE(bad_err.str().find("not supported"), std::string::npos) + << bad_err.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsExistingOutputWithoutTruncating) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_in", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,10\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_out", ".tsfile"); + { + std::ofstream o(out_path.c_str()); + o << "keep-me"; + } + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 3); + EXPECT_EQ(read_file(out_path), "keep-me"); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsNonRegularInputBeforeCreatingOutput) { +#ifndef _WIN32 + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_input_dir", ""); + ASSERT_EQ(mkdir(dir.c_str(), 0777), 0); + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_input_dir_out", ".tsfile"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", dir, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2); + EXPECT_FALSE(file_exists(out_path)); + + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirements, WriteRejectsNonCanonicalTimeLexemesAsInputErrors) { + const char* bad_times[] = {"+1", "01", "-0"}; + for (const char* bad_time : bad_times) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_bad_time", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n" << bad_time << ",10\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_bad_time_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 2) << bad_time << " " << err.str(); + EXPECT_FALSE(file_exists(out_path)) << bad_time; + std::remove(csv.c_str()); + std::remove(out_path.c_str()); + } +} + +TEST(CliRequirements, WriteTargetFailuresAreRuntimeErrors) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_target", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,1\n"; + } + + std::ostringstream same_out; + std::ostringstream same_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, + same_out, same_err), + 3); + EXPECT_TRUE(same_out.str().empty()); + EXPECT_NE(same_err.str().find("same as the input"), std::string::npos) + << same_err.str(); + EXPECT_EQ(read_file(csv), "time,s1\n0,1\n"); + + std::string parent = + tsfile_cli_test::unique_temp_path("tsfile_cli_missing_parent", ""); + std::string child = parent + "/out.tsfile"; + std::ostringstream parent_out; + std::ostringstream parent_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-i", csv, "-o", child}, + parent_out, parent_err), + 3); + EXPECT_TRUE(parent_out.str().empty()); + EXPECT_NE(parent_err.str().find("cannot create output"), std::string::npos) + << parent_err.str(); + EXPECT_FALSE(file_exists(child)); + + std::remove(csv.c_str()); +} + +TEST(CliRequirements, WriteCsvNullAndEmptyStringRemainDistinctTags) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\\N,21.0\n1000,\"\",22.0\n"; + } + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_tag_null_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 0) + << err.str(); + + std::ostringstream cat_out; + std::ostringstream cat_err; + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-t", "sensors", "-f", "ndjson", out_path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_NE(cat_out.str().find("\"site\":null"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"site\":\"\""), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":21"), std::string::npos) + << cat_out.str(); + EXPECT_NE(cat_out.str().find("\"temp\":22"), std::string::npos) + << cat_out.str(); + + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-t", "sensors", "-m", "site", "-f", + "csv", out_path}, + count_out, count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("table,sensors,site,TAG,2,2"), + std::string::npos) + << count_out.str(); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, WriteRejectsUnterminatedQuotedCsvField) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_unclosed_quote", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,site,temp\n1000,\"beijing,21.0\n"; + } + std::string out_path = tsfile_cli_test::unique_temp_path( + "tsfile_cli_unclosed_quote_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "sensors", "--tag", "site", "STRING", + "--field", "temp", "FLOAT", "-i", csv, "-o", out_path}, + out, err), + 2); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("unterminated quoted CSV field"), + std::string::npos) + << err.str(); + EXPECT_FALSE(file_exists(out_path)); + + std::remove(csv.c_str()); + std::remove(out_path.c_str()); +} + +TEST(CliRequirements, LegacyColumnsOptionIsRejected) { + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"write", "--table", "t1", "--columns", + "s1:INT64:field", "-o", "x.tsfile", "--stdin"}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("Unknown flag: --columns"), std::string::npos) + << err.str(); +} + +TEST(CliRequirements, WriteRejectsImplicitInputAndFormatFlag) { + std::ostringstream implicit_out; + std::ostringstream implicit_err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", + "INT64", "-o", "x.tsfile", "in.csv"}, + implicit_out, implicit_err), + 1); + EXPECT_NE( + implicit_err.str().find("choose exactly one of --input or --stdin"), + std::string::npos) + << implicit_err.str(); + + std::ostringstream format_out; + std::ostringstream format_err; + EXPECT_EQ( + tsfile_cli::run_cli({"write", "--table", "t1", "--field", "s1", "INT64", + "-f", "csv", "--stdin", "-o", "x.tsfile"}, + format_out, format_err), + 1); + EXPECT_NE(format_err.str().find("--format is not valid"), std::string::npos) + << format_err.str(); +} + +TEST(CliRequirements, ExportWritesSingleObjectAtomically) { + TableFixture f; + std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_export", ".csv"); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"export", "-t", "table1", "-o", out_path, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-t", "table1", "-m", "s1", "-f", "csv", f.path}, + cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_EQ(read_file(out_path), cat_out.str()); + + std::remove(out_path.c_str()); +} + +#ifndef _WIN32 +TEST(CliRequirements, ExportForceRejectsSymlinkAndSpecialTargets) { + TableFixture f; + std::string sentinel = + tsfile_cli_test::unique_temp_path("tsfile_cli_export_sentinel", ".txt"); + std::string symlink_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_export_link", ".csv"); + { + std::ofstream file(sentinel.c_str()); + file << "sentinel"; + } + ASSERT_EQ(symlink(sentinel.c_str(), symlink_path.c_str()), 0); + + std::ostringstream link_out; + std::ostringstream link_err; + EXPECT_EQ(tsfile_cli::run_cli({"export", "-t", "table1", "-o", symlink_path, + "--type", "csv", "--force", f.path}, + link_out, link_err), + 3); + EXPECT_EQ(read_file(sentinel), "sentinel"); + + std::string fifo_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_export_fifo", ".csv"); + ASSERT_EQ(mkfifo(fifo_path.c_str(), 0600), 0); + std::ostringstream fifo_out; + std::ostringstream fifo_err; + EXPECT_EQ(tsfile_cli::run_cli({"export", "-t", "table1", "-o", fifo_path, + "--type", "csv", "--force", f.path}, + fifo_out, fifo_err), + 3); + + std::remove(symlink_path.c_str()); + std::remove(sentinel.c_str()); + std::remove(fifo_path.c_str()); +} +#endif + +TEST(CliRequirements, ExportWritesMultiObjectManifestAndNumberedFiles) { + MultiTableFixture f; + std::string dir = + tsfile_cli_test::unique_temp_path("tsfile_cli_multi_export", ""); + + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"export", "-t", "sensors_a", "-t", "sensors_b", "--output-dir", dir, + "--type", "csv", "-m", "s1", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_EQ(read_file(dir + "/0001.csv"), + "time,id1,s1\n0,sensors_a_tag,10\n"); + EXPECT_EQ(read_file(dir + "/0002.csv"), + "time,id1,s1\n0,sensors_b_tag,20\n"); + std::string manifest = read_file(dir + "/_manifest.json"); + EXPECT_NE(manifest.find("\"complete\": true"), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"file\":\"0001.csv\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"object\":\"sensors_b\""), std::string::npos) + << manifest; + EXPECT_NE(manifest.find("\"rows\":\"1\""), std::string::npos) << manifest; + + std::remove((dir + "/0001.csv").c_str()); + std::remove((dir + "/0002.csv").c_str()); + std::remove((dir + "/_manifest.json").c_str()); +#ifndef _WIN32 + rmdir(dir.c_str()); +#endif +} + +TEST(CliRequirements, MultipleTagFiltersRequireAndHonorTagMatch) { + std::string path = tsfile_cli_test::write_tag_filter_fixture(); + std::ostringstream missing_out; + std::ostringstream missing_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "--tag-filter", "id1", "eq", "dev_a", + "--tag-filter", "id1", "eq", "dev_c", "-f", "csv", path}, + missing_out, missing_err), + 1); + + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", "eq", + "dev_a", "--tag-filter", "id1", "eq", "dev_c", + "--tag-match", "any", "-f", "csv", path}, + out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), "time,id1,s1\n0,dev_a,10\n3,dev_c,40\n"); + std::remove(path.c_str()); +} + +TEST(CliRequirements, SketchRejectsRegularResultFormat) { + TableFixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"sketch", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("sketch does not accept --format"), + std::string::npos) + << err.str(); +} + +TEST(CliRequirements, SketchWritesStdoutAndAtomicOutput) { + TableFixture f; + std::ostringstream stdout_out; + std::ostringstream stdout_err; + ASSERT_EQ(tsfile_cli::run_cli({"sketch", f.path}, stdout_out, stdout_err), + 0) + << stdout_err.str(); + EXPECT_NE(stdout_out.str().find("TsFile Sketch"), std::string::npos) + << stdout_out.str(); + EXPECT_NE(stdout_out.str().find("model: table"), std::string::npos) + << stdout_out.str(); + + std::string output = + tsfile_cli_test::unique_temp_path("tsfile_cli_sketch", ".txt"); + std::ostringstream file_out; + std::ostringstream file_err; + ASSERT_EQ(tsfile_cli::run_cli({"sketch", "-o", output, f.path}, file_out, + file_err), + 0) + << file_err.str(); + EXPECT_TRUE(file_out.str().empty()); + EXPECT_EQ(read_file(output), stdout_out.str()); + + const std::string sentinel = "do not replace"; + { + std::ofstream existing(output.c_str(), std::ios::binary); + existing << sentinel; + } + std::ostringstream no_force_out; + std::ostringstream no_force_err; + EXPECT_EQ(tsfile_cli::run_cli({"sketch", "-o", output, f.path}, + no_force_out, no_force_err), + 3); + EXPECT_EQ(read_file(output), sentinel); + + std::ostringstream force_out; + std::ostringstream force_err; + ASSERT_EQ(tsfile_cli::run_cli({"sketch", "-o", output, "--force", f.path}, + force_out, force_err), + 0) + << force_err.str(); + EXPECT_EQ(read_file(output), stdout_out.str()); + + const std::string source_before = read_file(f.path); + std::ostringstream alias_out; + std::ostringstream alias_err; + EXPECT_EQ(tsfile_cli::run_cli({"sketch", "-o", f.path, "--force", f.path}, + alias_out, alias_err), + 3); + EXPECT_EQ(read_file(f.path), source_before); + std::remove(output.c_str()); +} + +TEST(CliRequirements, ModelDetectionPrefersTableSchemaAndFallsBackToTree) { + TableFixture table; + storage::TsFileReader table_reader; + ASSERT_EQ(table_reader.open(table.path), common::E_OK); + EXPECT_FALSE(table_reader.get_all_table_schemas().empty()); + EXPECT_FALSE(table_reader.get_all_device_ids().empty()); + table_reader.close(); + + std::ostringstream table_out; + std::ostringstream table_err; + ASSERT_EQ(tsfile_cli::run_cli({"ls", "-f", "csv", table.path}, table_out, + table_err), + 0) + << table_err.str(); + EXPECT_EQ(table_out.str(), "model,object\ntable,table1\n"); + + std::string tree_path = tsfile_cli_test::write_sparse_tree_fixture(); + storage::TsFileReader tree_reader; + ASSERT_EQ(tree_reader.open(tree_path), common::E_OK); + EXPECT_TRUE(tree_reader.get_all_table_schemas().empty()); + EXPECT_FALSE(tree_reader.get_all_device_ids().empty()); + tree_reader.close(); + + std::ostringstream tree_out; + std::ostringstream tree_err; + ASSERT_EQ( + tsfile_cli::run_cli({"ls", "-f", "csv", tree_path}, tree_out, tree_err), + 0) + << tree_err.str(); + EXPECT_EQ(tree_out.str(), "model,object\ntree,root.test.d1\n"); + std::remove(tree_path.c_str()); +} diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 5b4e532d9..1f88af8c5 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -27,13 +27,16 @@ #include #endif +#include #include #include +#include #include "common/schema.h" #include "common/tablet.h" #include "file/write_file.h" #include "writer/tsfile_table_writer.h" +#include "writer/tsfile_tree_writer.h" namespace tsfile_cli_test { @@ -142,6 +145,181 @@ inline std::string write_tag_filter_fixture() { return out_path; } +inline std::string write_nullable_tag_filter_fixture() { + storage::libtsfile_init(); + std::string out_path = + unique_temp_path("tsfile_cli_nullable_tag_filter_fixture", ".tsfile"); + std::string table_name = "t1"; + + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(out_path, flags, 0666); + + auto* schema = new storage::TableSchema( + table_name, + { + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("s1", common::INT64, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD), + }); + + auto* writer = new storage::TsFileTableWriter(&file, schema); + storage::Tablet tablet( + table_name, {"id1", "s1"}, {common::STRING, common::INT64}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 5); + const char* tags[] = {nullptr, "", "null", "dev_a", "dev_b"}; + for (int row = 0; row < 5; ++row) { + tablet.add_timestamp(row, static_cast(row)); + if (tags[row] != nullptr) { + tablet.add_value(row, "id1", tags[row]); + } + tablet.add_value(row, "s1", static_cast((row + 1) * 10)); + } + + writer->write_table(tablet); + writer->flush(); + writer->close(); + + delete writer; + delete schema; + return out_path; +} + +inline void write_one_table_row(storage::TsFileTableWriter* writer, + const std::string& table_name, + const std::string& field_name, int64_t time, + int64_t value) { + storage::Tablet tablet( + table_name, {"id1", field_name}, {common::STRING, common::INT64}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); + tablet.add_timestamp(0, time); + tablet.add_value(0, "id1", table_name + "_tag"); + tablet.add_value(0, field_name, value); + writer->write_table(tablet); +} + +inline std::string write_two_table_fixture(const std::string& fixture_name, + const std::string& first_field, + const std::string& second_field) { + storage::libtsfile_init(); + std::string out_path = unique_temp_path(fixture_name, ".tsfile"); + + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(out_path, flags, 0666); + + auto* schema_a = new storage::TableSchema( + "sensors_a", + { + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema(first_field, common::INT64, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::FIELD), + }); + auto* writer = new storage::TsFileTableWriter(&file, schema_a); + auto schema_b = std::make_shared( + "sensors_b", + std::vector{ + common::ColumnSchema("id1", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema(second_field, common::INT64, + common::UNCOMPRESSED, common::PLAIN, + common::ColumnCategory::FIELD), + }); + writer->register_table(schema_b); + + write_one_table_row(writer, "sensors_a", first_field, 0, 10); + write_one_table_row(writer, "sensors_b", second_field, 0, 20); + + writer->flush(); + writer->close(); + + delete writer; + delete schema_a; + return out_path; +} + +inline std::string write_multi_table_fixture() { + return write_two_table_fixture("tsfile_cli_multi_table_fixture", "s1", + "s1"); +} + +inline std::string write_disjoint_table_fixture() { + return write_two_table_fixture("tsfile_cli_disjoint_table_fixture", "s1", + "s2"); +} + +inline std::string write_sparse_tree_fixture() { + storage::libtsfile_init(); + std::string path = unique_temp_path("tsfile_cli_sparse_tree", ".tsfile"); + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(path, flags, 0666); + storage::TsFileTreeWriter writer(&file); + std::string device = "root.test.d1"; + auto* left = new storage::MeasurementSchema("left", common::INT32); + auto* right = new storage::MeasurementSchema("right", common::BOOLEAN); + writer.register_timeseries(device, left); + writer.register_timeseries(device, right); + + storage::TsRecord first(device, 0); + first.add_point("left", static_cast(10)); + writer.write(first); + storage::TsRecord second(device, 1); + second.add_point("right", true); + writer.write(second); + storage::TsRecord third(device, 2); + third.add_point("left", static_cast(20)); + third.add_point("right", false); + writer.write(third); + writer.flush(); + writer.close(); + delete left; + delete right; + return path; +} + +inline std::string write_disjoint_tree_fixture() { + storage::libtsfile_init(); + std::string path = unique_temp_path("tsfile_cli_disjoint_tree", ".tsfile"); + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(path, flags, 0666); + storage::TsFileTreeWriter writer(&file); + auto* left = new storage::MeasurementSchema("left", common::INT32); + auto* right = new storage::MeasurementSchema("right", common::BOOLEAN); + std::string first_device = "root.test.d1"; + std::string second_device = "root.test.d2"; + writer.register_timeseries(first_device, left); + writer.register_timeseries(second_device, right); + + storage::TsRecord first(first_device, 0); + first.add_point("left", static_cast(10)); + writer.write(first); + storage::TsRecord second(second_device, 1); + second.add_point("right", true); + writer.write(second); + writer.flush(); + writer.close(); + delete left; + delete right; + return path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index a5c5b722c..16fe7694f 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -39,6 +39,31 @@ struct TagFilterFixture { ~TagFilterFixture() { std::remove(path.c_str()); } }; +struct MultiTableFixture { + std::string path = tsfile_cli_test::write_multi_table_fixture(); + ~MultiTableFixture() { std::remove(path.c_str()); } +}; + +struct DisjointTableFixture { + std::string path = tsfile_cli_test::write_disjoint_table_fixture(); + ~DisjointTableFixture() { std::remove(path.c_str()); } +}; + +struct SparseTreeFixture { + std::string path = tsfile_cli_test::write_sparse_tree_fixture(); + ~SparseTreeFixture() { std::remove(path.c_str()); } +}; + +struct DisjointTreeFixture { + std::string path = tsfile_cli_test::write_disjoint_tree_fixture(); + ~DisjointTreeFixture() { std::remove(path.c_str()); } +}; + +class FlushFailingStreamBuf : public std::stringbuf { + protected: + int sync() override { return -1; } +}; + size_t count_lines(const std::string& s) { size_t n = 0; for (char c : s) { @@ -55,20 +80,21 @@ TEST(CliE2E, LsListsTableNameTsv) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"ls", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "name\ntable1\n"); + EXPECT_EQ(out.str(), "model,object\ntable,table1\n"); EXPECT_TRUE(err.str().empty()); } -TEST(CliE2E, LsNoHeaderJustName) { +TEST(CliE2E, LsRejectsNoHeader) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"ls", "-f", "tsv", "--no-header", f.path}, + int code = tsfile_cli::run_cli({"ls", "-f", "csv", "--no-header", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "table1\n"); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--no-header"), std::string::npos) << err.str(); } TEST(CliE2E, OpenMissingFileReturnsFileError) { @@ -84,11 +110,11 @@ TEST(CliE2E, SchemaShowsFieldColumnAndType) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE( - out.str().find("target\tmeasurement\tdatatype\tencoding\tcompression"), - std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,data_type,encoding," + "compression"), + std::string::npos); EXPECT_NE(out.str().find("s1"), std::string::npos); EXPECT_NE(out.str().find("INT64"), std::string::npos); } @@ -97,25 +123,51 @@ TEST(CliE2E, SchemaTableMeasurementFilterOnlyShowsRequestedColumn) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "tsv", f.path}, + int code = tsfile_cli::run_cli({"schema", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("table1\ts1\tINT64"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid1"), std::string::npos); - EXPECT_EQ(out.str().find("table1\tid2"), std::string::npos); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id1"), std::string::npos); + EXPECT_EQ(out.str().find("table1,id2"), std::string::npos); +} + +TEST(CliE2E, SchemaTableMeasurementFilterIsCaseInsensitive) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"schema", "-m", "S1", "-f", "csv", f.path}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,INT64"), std::string::npos) + << out.str(); +} + +TEST(CliE2E, SchemaRejectsMissingTableBeforeWritingOutput) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"schema", "-t", "missing", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("does not exist"), std::string::npos) << err.str(); } TEST(CliE2E, StatsReportsCountAndTimeRange) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"stats", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("target\tmeasurement\tcount\tstart_time\tend_" - "time\tmin\tmax\tfirst\tlast\tsum"), - std::string::npos); - EXPECT_NE(out.str().find("s1\t5\t0\t4\t0\t40\t0\t40\t100"), - std::string::npos); + EXPECT_NE(out.str().find("model,object,tag.id1,tag.id2,field,data_type," + "non_null_count,null_count,min_time,max_time,min," + "max,first,last,sum,stats_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1_field_1,id2_field_2,s1,INT64," + "5,\\N,0,4,0,40,0,40,\\N,statistics"), + std::string::npos) + << out.str(); } TEST(CliE2E, HeadProjectsAndLimits) { @@ -123,9 +175,11 @@ TEST(CliE2E, HeadProjectsAndLimits) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "-n", "2", "-f", "tsv", f.path}, out, err); + {"head", "-m", "s1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n0\t0\n1\t10\n"); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n0,id1_field_1,id2_field_2,0\n1,id1_field_1," + "id2_field_2,10\n"); } TEST(CliE2E, CatReturnsAllRows) { @@ -133,10 +187,37 @@ TEST(CliE2E, CatReturnsAllRows) { std::ostringstream out; std::ostringstream err; int code = - tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", f.path}, out, err); + tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_EQ(count_lines(out.str()), 6u); - EXPECT_NE(out.str().find("time\ts1\n"), std::string::npos); + EXPECT_NE(out.str().find("time,id1,id2,s1\n"), std::string::npos); +} + +TEST(CliE2E, CatRejectsMissingOrNonFieldProjectionBeforeQuery) { + Fixture f; + for (const char* measurement : {"missing", "id1"}) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"cat", "-m", measurement, "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 1) << measurement << " " << err.str(); + EXPECT_TRUE(out.str().empty()) << measurement; + EXPECT_NE(err.str().find("FIELD"), std::string::npos) + << measurement << " " << err.str(); + } +} + +TEST(CliE2E, CatTableProjectionIsCaseInsensitive) { + Fixture f; + std::ostringstream out; + std::ostringstream err; + int code = + tsfile_cli::run_cli({"cat", "-m", "S1", "-f", "csv", f.path}, out, err); + EXPECT_EQ(code, 0) << err.str(); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n0,id1_field_1,id2_field_2,0\n1,id1_field_1," + "id2_field_2,10\n2,id1_field_1,id2_field_2,20\n3,id1_field_1," + "id2_field_2,30\n4,id1_field_1,id2_field_2,40\n"); } TEST(CliE2E, CatPushesDownOffsetAndLimit) { @@ -144,10 +225,12 @@ TEST(CliE2E, CatPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--offset", "2", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n2,id1_field_1,id2_field_2,20\n3,id1_field_1," + "id2_field_2,30\n"); } TEST(CliE2E, HeadPushesDownOffsetAndLimit) { @@ -155,10 +238,12 @@ TEST(CliE2E, HeadPushesDownOffsetAndLimit) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "tsv", f.path}, + {"head", "-m", "s1", "--offset", "1", "-n", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n1\t10\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n1,id1_field_1,id2_field_2,10\n2,id1_field_1," + "id2_field_2,20\n3,id1_field_1,id2_field_2,30\n"); } TEST(CliE2E, CatWithTimeRange) { @@ -166,10 +251,12 @@ TEST(CliE2E, CatWithTimeRange) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "tsv", f.path}, + {"cat", "-m", "s1", "--start", "2", "--end", "3", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n2,id1_field_1,id2_field_2,20\n3,id1_field_1," + "id2_field_2,30\n"); } TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { @@ -178,10 +265,12 @@ TEST(CliE2E, CatAppliesOffsetAfterTimeRange) { std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "1", "--end", "4", - "--offset", "1", "-n", "2", "-f", "tsv", f.path}, + "--offset", "1", "-n", "2", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "time\ts1\n2\t20\n3\t30\n"); + EXPECT_EQ(out.str(), + "time,id1,id2,s1\n2,id1_field_1,id2_field_2,20\n3,id1_field_1," + "id2_field_2,30\n"); } TEST(CliE2E, CatFiltersRowsByTagEq) { @@ -189,34 +278,22 @@ TEST(CliE2E, CatFiltersRowsByTagEq) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", - "eq", "dev_b", "-f", "tsv", f.path}, + "eq", "dev_b", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,id1,s1\n1,dev_b,20\n2,dev_b,30\n"); } -TEST(CliE2E, HeadFiltersRowsByTagBetween) { - TagFilterFixture f; - std::ostringstream out; - std::ostringstream err; - int code = - tsfile_cli::run_cli({"head", "-m", "s1", "--tag-between", "id1", - "dev_b", "dev_c", "-n", "10", "-f", "tsv", f.path}, - out, err); - EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n3\t40\n"); -} - -TEST(CliE2E, SampleFiltersRowsByTagEq) { +TEST(CliE2E, HeadFiltersRowsByTagRegexp) { TagFilterFixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"sample", "-m", "s1", "--tag-filter", "id1", "eq", "dev_b", "-n", "10", - "--seed", "1", "-f", "tsv", f.path}, + {"head", "-m", "s1", "--tag-filter", "id1", "regexp", "dev_[bc]", "-n", + "10", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_EQ(out.str(), "time\ts1\n1\t20\n2\t30\n"); + EXPECT_EQ(out.str(), "time,id1,s1\n1,dev_b,20\n2,dev_b,30\n3,dev_c,40\n"); } TEST(CliE2E, TagFilterRejectsFieldColumn) { @@ -224,47 +301,363 @@ TEST(CliE2E, TagFilterRejectsFieldColumn) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "s1", - "eq", "20", "-f", "tsv", f.path}, + "eq", "20", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("invalid tag filter column"), std::string::npos) << err.str(); } +TEST(CliE2E, TagFiltersDistinguishNullEmptyAndLiteralNull) { + std::string path = tsfile_cli_test::write_nullable_tag_filter_fixture(); + + struct Case { + std::vector filter; + std::string expected_row; + }; + const std::vector cases = { + {{"id1", "is-null"}, "0,\\N,10\n"}, + {{"id1", "eq", ""}, "1,\"\",20\n"}, + {{"id1", "eq", "null"}, "2,null,30\n"}, + }; + for (const Case& test_case : cases) { + std::vector args = {"cat", "-m", "s1", "--tag-filter"}; + args.insert(args.end(), test_case.filter.begin(), + test_case.filter.end()); + args.insert(args.end(), {"-f", "csv", path}); + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ(tsfile_cli::run_cli(args, out, err), 0) << err.str(); + EXPECT_EQ(out.str(), "time,id1,s1\n" + test_case.expected_row); + } + + std::ostringstream not_null_out; + std::ostringstream not_null_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "not-null", "-f", "csv", path}, + not_null_out, not_null_err), + 0) + << not_null_err.str(); + EXPECT_EQ(not_null_out.str().find("0,\\N,10"), std::string::npos) + << not_null_out.str(); + EXPECT_NE(not_null_out.str().find("1,\"\",20"), std::string::npos) + << not_null_out.str(); + + std::ostringstream neq_out; + std::ostringstream neq_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "neq", "dev_a", "-f", "csv", path}, + neq_out, neq_err), + 0) + << neq_err.str(); + EXPECT_EQ(neq_out.str().find("0,\\N,10"), std::string::npos) + << neq_out.str(); + EXPECT_EQ(neq_out.str().find("3,dev_a,40"), std::string::npos) + << neq_out.str(); + EXPECT_NE(neq_out.str().find("4,dev_b,50"), std::string::npos) + << neq_out.str(); + + std::remove(path.c_str()); +} + +TEST(CliE2E, TagRegexpUsesFullValueAndRejectsInvalidPatterns) { + std::string path = tsfile_cli_test::write_nullable_tag_filter_fixture(); + + std::ostringstream substring_out; + std::ostringstream substring_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "regexp", "dev", "-f", "csv", path}, + substring_out, substring_err), + 0) + << substring_err.str(); + EXPECT_EQ(substring_out.str(), "time,id1,s1\n"); + + std::ostringstream full_out; + std::ostringstream full_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "regexp", "dev_.*", "-f", "csv", path}, + full_out, full_err), + 0) + << full_err.str(); + EXPECT_NE(full_out.str().find("3,dev_a,40"), std::string::npos) + << full_out.str(); + EXPECT_NE(full_out.str().find("4,dev_b,50"), std::string::npos) + << full_out.str(); + + std::ostringstream invalid_out; + std::ostringstream invalid_err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", + "regexp", "[", "-f", "csv", path}, + invalid_out, invalid_err), + 1); + EXPECT_TRUE(invalid_out.str().empty()); + EXPECT_NE(invalid_err.str().find("invalid regular expression"), + std::string::npos) + << invalid_err.str(); + + std::remove(path.c_str()); +} + +TEST(CliE2E, MultipleTagFiltersHonorAll) { + std::string path = tsfile_cli_test::write_nullable_tag_filter_fixture(); + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-m", "s1", "--tag-filter", "id1", "neq", + "dev_a", "--tag-filter", "id1", "regexp", "dev_.*", + "--tag-match", "all", "-f", "csv", path}, + out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), "time,id1,s1\n4,dev_b,50\n"); + std::remove(path.c_str()); +} + +TEST(CliE2E, RowWindowDistinguishesExactAndExcessiveOffset) { + Fixture f; + std::ostringstream exact_out; + std::ostringstream exact_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "--offset", "5", "-f", "csv", f.path}, + exact_out, exact_err), + 0) + << exact_err.str(); + EXPECT_EQ(exact_out.str(), "time,id1,id2,s1\n"); + + std::ostringstream excessive_out; + std::ostringstream excessive_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "--offset", "6", "-f", "csv", f.path}, + excessive_out, excessive_err), + 1); + EXPECT_TRUE(excessive_out.str().empty()); + EXPECT_NE(excessive_err.str().find("offset exceeds matched row count"), + std::string::npos) + << excessive_err.str(); +} + +TEST(CliE2E, ZeroLimitUsesEachFormatsZeroRowContract) { + Fixture f; + std::ostringstream csv_out; + std::ostringstream csv_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-m", "s1", "-n", "0", "-f", "csv", f.path}, + csv_out, csv_err), + 0) + << csv_err.str(); + EXPECT_EQ(csv_out.str(), "time,id1,id2,s1\n"); + + std::ostringstream ndjson_out; + std::ostringstream ndjson_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "-n", "0", "-f", "ndjson", f.path}, + ndjson_out, ndjson_err), + 0) + << ndjson_err.str(); + EXPECT_TRUE(ndjson_out.str().empty()); + + std::ostringstream table_out; + std::ostringstream table_err; + EXPECT_EQ(tsfile_cli::run_cli( + {"cat", "-m", "s1", "-n", "0", "-f", "table", f.path}, + table_out, table_err), + 0) + << table_err.str(); + EXPECT_NE(table_out.str().find("time"), std::string::npos) + << table_out.str(); + EXPECT_NE(table_out.str().find("s1"), std::string::npos) << table_out.str(); + EXPECT_EQ(table_out.str().find("id1_field_1"), std::string::npos) + << table_out.str(); +} + TEST(CliE2E, CatJsonIsNdjson) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"cat", "-m", "s1", "--start", "0", "--end", "0", "-f", "json", f.path}, - out, err); + int code = tsfile_cli::run_cli({"cat", "-m", "s1", "--start", "0", "--end", + "0", "-f", "ndjson", f.path}, + out, err); EXPECT_EQ(code, 0); - EXPECT_EQ(out.str(), "{\"time\":0,\"s1\":0}\n"); + EXPECT_EQ(out.str(), + "{\"time\":\"0\",\"id1\":\"id1_field_1\",\"id2\":" + "\"id2_field_2\",\"s1\":\"0\"}\n"); +} + +TEST(CliE2E, StdoutFlushFailureReturnsRuntimeError) { + Fixture f; + FlushFailingStreamBuf buffer; + std::ostream out(&buffer); + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", f.path}, out, err), 3); + EXPECT_NE(err.str().find("failed to read rows"), std::string::npos) + << err.str(); } TEST(CliE2E, MetaReportsFileSummary) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"meta", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("file\tmodel\tdevice_count\ttable_count\tseries_" - "count\tstart_time\tend_time\tfile_size_bytes"), - std::string::npos); - EXPECT_NE(out.str().find("\ttable\t"), std::string::npos); + EXPECT_NE(out.str().find("size_bytes,format_version,model\n"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find(",4,table\n"), std::string::npos) << out.str(); +} + +TEST(CliE2E, MetaRejectsUnsupportedFileHeaderVersion) { + Fixture f; + std::fstream file(f.path.c_str(), + std::ios::in | std::ios::out | std::ios::binary); + ASSERT_TRUE(file.good()); + file.seekp(storage::MAGIC_STRING_TSFILE_LEN); + file.put(static_cast(storage::VERSION_NUM_BYTE + 1)); + file.close(); + + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"meta", "-f", "csv", f.path}, out, err), 2); + EXPECT_TRUE(out.str().empty()); } -TEST(CliE2E, CountReportsSeriesCountsAndTotal) { +TEST(CliE2E, CountReportsColumnCountsWithoutSummaryRows) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"count", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"count", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); EXPECT_TRUE(err.str().empty()); - EXPECT_NE(out.str().find("target\tmeasurement\tcount"), std::string::npos); - EXPECT_NE(out.str().find("\ts1\t5"), std::string::npos); - EXPECT_NE(out.str().find("total\t\t"), std::string::npos); + EXPECT_NE(out.str().find("model,object,column,category,row_count,entity_" + "count,non_null_count,null_count,min_time," + "max_time,time_source"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,id1,TAG,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_NE(out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), + std::string::npos) + << out.str(); + EXPECT_EQ(out.str().find("total"), std::string::npos) << out.str(); +} + +TEST(CliE2E, CountAndStatsDefaultToAllTables) { + MultiTableFixture f; + std::ostringstream count_out; + std::ostringstream count_err; + EXPECT_EQ(tsfile_cli::run_cli({"count", "-f", "csv", f.path}, count_out, + count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("table,sensors_a,s1,FIELD,1,1,1,0,0,0,scan"), + std::string::npos) + << count_out.str(); + EXPECT_NE(count_out.str().find("table,sensors_b,s1,FIELD,1,1,1,0,0,0,scan"), + std::string::npos) + << count_out.str(); + + std::ostringstream stats_out; + std::ostringstream stats_err; + EXPECT_EQ(tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, stats_out, + stats_err), + 0) + << stats_err.str(); + EXPECT_NE(stats_out.str().find("table,sensors_a,sensors_a_tag,s1,INT64"), + std::string::npos) + << stats_out.str(); + EXPECT_NE(stats_out.str().find("table,sensors_b,sensors_b_tag,s1,INT64"), + std::string::npos) + << stats_out.str(); +} + +TEST(CliE2E, CountAndStatsProjectionMayMatchOnlySomeTables) { + DisjointTableFixture f; + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-m", "S1", "-f", "csv", f.path}, + count_out, count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("table,sensors_a,s1,FIELD"), + std::string::npos) + << count_out.str(); + EXPECT_EQ(count_out.str().find("table,sensors_b"), std::string::npos) + << count_out.str(); + + std::ostringstream stats_out; + std::ostringstream stats_err; + ASSERT_EQ(tsfile_cli::run_cli({"stats", "-m", "S1", "-f", "csv", f.path}, + stats_out, stats_err), + 0) + << stats_err.str(); + EXPECT_NE(stats_out.str().find("table,sensors_a,sensors_a_tag,s1,INT64"), + std::string::npos) + << stats_out.str(); + EXPECT_EQ(stats_out.str().find("table,sensors_b"), std::string::npos) + << stats_out.str(); +} + +TEST(CliE2E, CountAndStatsProjectionMayMatchOnlySomeTreeDevices) { + DisjointTreeFixture f; + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-m", "left", "-f", "csv", f.path}, + count_out, count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find("tree,root.test.d1,left,FIELD"), + std::string::npos) + << count_out.str(); + EXPECT_EQ(count_out.str().find("tree,root.test.d2"), std::string::npos) + << count_out.str(); + + std::ostringstream stats_out; + std::ostringstream stats_err; + ASSERT_EQ(tsfile_cli::run_cli({"stats", "-m", "left", "-f", "csv", f.path}, + stats_out, stats_err), + 0) + << stats_err.str(); + EXPECT_NE(stats_out.str().find("tree,root.test.d1,left,INT32"), + std::string::npos) + << stats_out.str(); + EXPECT_EQ(stats_out.str().find("tree,root.test.d2"), std::string::npos) + << stats_out.str(); +} + +TEST(CliE2E, TreeCountAndStatsUseDeviceTimestampUnion) { + SparseTreeFixture f; + std::ostringstream count_out; + std::ostringstream count_err; + ASSERT_EQ(tsfile_cli::run_cli({"count", "-f", "csv", f.path}, count_out, + count_err), + 0) + << count_err.str(); + EXPECT_NE(count_out.str().find( + "tree,root.test.d1,left,FIELD,3,\\N,2,1,0,2,statistics"), + std::string::npos) + << count_out.str(); + EXPECT_NE(count_out.str().find( + "tree,root.test.d1,right,FIELD,3,\\N,2,1,1,2,statistics"), + std::string::npos) + << count_out.str(); + + std::ostringstream stats_out; + std::ostringstream stats_err; + ASSERT_EQ(tsfile_cli::run_cli({"stats", "-f", "csv", f.path}, stats_out, + stats_err), + 0) + << stats_err.str(); + EXPECT_NE(stats_out.str().find( + "tree,root.test.d1,left,INT32,2,1,0,2,10,20,10,20,30," + "statistics"), + std::string::npos) + << stats_out.str(); + EXPECT_NE(stats_out.str().find( + "tree,root.test.d1,right,BOOLEAN,2,1,1,2,\\N,\\N,true,false," + "1,statistics"), + std::string::npos) + << stats_out.str(); } TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { @@ -273,56 +666,35 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { std::ostringstream schema_out; std::ostringstream schema_err; EXPECT_EQ( - tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"schema", "-t", "TABLE1", "-f", "csv", f.path}, schema_out, schema_err), 0); - EXPECT_NE(schema_out.str().find("table1\ts1\tINT64"), std::string::npos) + EXPECT_NE(schema_out.str().find("table,table1,s1,FIELD,INT64"), + std::string::npos) << schema_out.str(); std::ostringstream count_out; std::ostringstream count_err; EXPECT_EQ( - tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"count", "-t", "TABLE1", "-f", "csv", f.path}, count_out, count_err), 0); - EXPECT_NE(count_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(count_out.str().find("table,table1,s1,FIELD,5,1,5,0,0,4,scan"), std::string::npos) << count_out.str(); std::ostringstream stats_out; std::ostringstream stats_err; EXPECT_EQ( - tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "tsv", f.path}, + tsfile_cli::run_cli({"stats", "-t", "TABLE1", "-f", "csv", f.path}, stats_out, stats_err), 0); - EXPECT_NE(stats_out.str().find("table1.id1_field_1.id2_field_2\ts1\t5"), + EXPECT_NE(stats_out.str().find("table,table1,id1_field_1,id2_field_2,s1," + "INT64,5,\\N,0,4,0,40,0,40,\\N,statistics"), std::string::npos) << stats_out.str(); } -TEST(CliE2E, SampleIsReproducibleWithSeed) { - Fixture f; - std::ostringstream out1; - std::ostringstream err1; - std::ostringstream out2; - std::ostringstream err2; - - int code1 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out1, err1); - int code2 = tsfile_cli::run_cli( - {"sample", "-m", "s1", "-n", "3", "--seed", "7", "-f", "tsv", f.path}, - out2, err2); - - EXPECT_EQ(code1, 0); - EXPECT_EQ(code2, 0); - EXPECT_TRUE(err1.str().empty()); - EXPECT_TRUE(err2.str().empty()); - EXPECT_EQ(out1.str(), out2.str()); - EXPECT_EQ(count_lines(out1.str()), 4u); - EXPECT_NE(out1.str().find("time\ts1\n"), std::string::npos); -} - TEST(CliE2E, WriteThenReadRoundTrip) { std::string csv_path = tsfile_cli_test::unique_temp_path("tsfile_cli_write_in", ".csv"); @@ -336,24 +708,26 @@ TEST(CliE2E, WriteThenReadRoundTrip) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", "id1:STRING:tag,s1:INT64:field", - "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "s1", + "INT64", "-i", csv_path, "-o", out_path}, wout, werr); EXPECT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; int cc = - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); EXPECT_EQ(cc, 0); - EXPECT_NE(cout_.str().find("\ts1\t3"), std::string::npos) << cout_.str(); + EXPECT_NE(cout_.str().find("table,t1,s1,FIELD,3,1,3,0,0,2,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "tsv", out_path}, + int rc = tsfile_cli::run_cli({"cat", "-m", "s1", "-f", "csv", out_path}, rout, rerr); EXPECT_EQ(rc, 0); - EXPECT_EQ(rout.str(), "time\ts1\n0\t0\n1\t10\n2\t20\n"); + EXPECT_EQ(rout.str(), "time,id1,s1\n0,dev,0\n1,dev,10\n2,dev,20\n"); std::remove(csv_path.c_str()); std::remove(out_path.c_str()); @@ -372,16 +746,15 @@ TEST(CliE2E, WriteThenReadFloatDoubleRoundTripLossless) { std::ostringstream wout; std::ostringstream werr; - int wc = - tsfile_cli::run_cli({"write", "--table", "t1", "--columns", - "id1:STRING:tag,f1:FLOAT:field,d1:DOUBLE:field", - "-o", out_path, csv_path}, - wout, werr); + int wc = tsfile_cli::run_cli( + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "f1", + "FLOAT", "--field", "d1", "DOUBLE", "-i", csv_path, "-o", out_path}, + wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - int rc = tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr); + int rc = tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr); ASSERT_EQ(rc, 0) << rerr.str(); // Default ostream precision (6 sig digits) would print 0.1 / 3.40282 and // lose bits; max_digits10 keeps every digit needed to round-trip. @@ -409,21 +782,23 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,note:TEXT:field", "-o", out_path, csv_path}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "note", + "TEXT", "-i", csv_path, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tnote\t2"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find("table,t1,note,FIELD,2,1,2,0,0,1,scan"), + std::string::npos) + << cout_.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0); + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0); EXPECT_NE(rout.str().find("line one\\nline two"), std::string::npos) << rout.str(); @@ -437,7 +812,7 @@ TEST(CliE2E, WriteMissingColumnsIsUsageError) { int code = tsfile_cli::run_cli( {"write", "--table", "t1", "-o", "x.tsfile", "in.csv"}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("--columns"), std::string::npos); + EXPECT_NE(err.str().find("--field"), std::string::npos); } namespace { @@ -459,10 +834,10 @@ TEST(CliE2E, WriteRejectsOutOfOrderTimestampsAndLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_NE(err.str().find("line 3"), std::string::npos) << err.str(); @@ -485,15 +860,17 @@ TEST(CliE2E, WriteAllowsSameTimestampAcrossDevices) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,s1:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("total\t\t3"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find("table,t,s1,FIELD,3,2,3,0,1,2,scan"), + std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -508,10 +885,10 @@ TEST(CliE2E, WriteRejectsOutputEqualsInput) { } std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", csv, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", csv}, out, err); - EXPECT_EQ(code, 1); + EXPECT_EQ(code, 3); EXPECT_NE(err.str().find("same as the input"), std::string::npos) << err.str(); // The input file must be untouched. @@ -535,10 +912,10 @@ TEST(CliE2E, WriteFailureOnBadValueLeavesNoOutput) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_FALSE(path_exists(out_path)); std::remove(csv.c_str()); @@ -549,20 +926,70 @@ TEST(CliE2E, WriteRejectsDuplicateColumnNames) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field,s1:INT64:field", - "-o", "x.tsfile", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "--field", "s1", + "INT64", "--stdin", "-o", "x.tsfile"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("duplicate column"), std::string::npos) << err.str(); } +TEST(CliE2E, WriteRejectsTagOnlySchema) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli({"write", "--table", "t", "--tag", "id", + "STRING", "--stdin", "-o", "x.tsfile"}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("--field"), std::string::npos) << err.str(); +} + +TEST(CliE2E, WriteRejectsNonStringTag) { + std::ostringstream out; + std::ostringstream err; + int code = tsfile_cli::run_cli( + {"write", "--table", "t", "--tag", "id", "INT64", "--field", "s1", + "INT64", "--stdin", "-o", "x.tsfile"}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("must use STRING"), std::string::npos) + << err.str(); +} + +TEST(CliE2E, WriteNormalizesNamesAndHeaderCase) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_case", ".csv"); + { + std::ofstream o(csv.c_str()); + o << "time,ID,S1\n0,a,1\n"; + } + std::string output = + tsfile_cli_test::unique_temp_path("tsfile_cli_case_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli( + {"write", "--table", "Mixed", "--tag", "Id", "STRING", + "--field", "s1", "INT64", "-i", csv, "-o", output}, + out, err), + 0) + << err.str(); + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ(tsfile_cli::run_cli({"schema", "-f", "csv", output}, schema_out, + schema_err), + 0); + EXPECT_NE(schema_out.str().find("table,mixed,id,TAG,STRING"), + std::string::npos); + std::remove(csv.c_str()); + std::remove(output.c_str()); +} + TEST(CliE2E, WriteRejectsHeaderMatchWithNoHeader) { std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", "-o", - "x.tsfile", "--no-header", "--header-match", "-"}, + {"write", "--table", "t", "--field", "s1", "INT64", "-o", "x.tsfile", + "--stdin", "--no-header", "--header-match"}, out, err); EXPECT_EQ(code, 1); EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); @@ -592,11 +1019,11 @@ TEST(CliE2E, SchemaTableShowsEncodingAndCompression) { Fixture f; std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"schema", "-f", "tsv", f.path}, out, err); + int code = tsfile_cli::run_cli({"schema", "-f", "csv", f.path}, out, err); EXPECT_EQ(code, 0); // Table-model schema must report the fixture's configured encoding and // compression rather than blanks. - EXPECT_NE(out.str().find("\ts1\tINT64\tPLAIN\tUNCOMPRESSED\n"), + EXPECT_NE(out.str().find(",s1,FIELD,INT64,PLAIN,UNCOMPRESSED\n"), std::string::npos) << out.str(); } @@ -616,10 +1043,9 @@ int write_one_value(const std::string& type, const std::string& value, tsfile_cli_test::unique_temp_path("tsfile_cli_ovf_out", ".tsfile"); std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:" + type + ":field", "-o", out_path, csv}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + type, "-i", csv, "-o", out_path}, + out, err); err_out = err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -629,7 +1055,7 @@ int write_one_value(const std::string& type, const std::string& value, TEST(CliE2E, WriteRejectsInt32Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT32", "3000000000", err), 3); + EXPECT_EQ(write_one_value("INT32", "3000000000", err), 2); EXPECT_NE(err.find("INT32 out of range"), std::string::npos) << err; } @@ -640,22 +1066,130 @@ TEST(CliE2E, WriteAcceptsInt32Boundary) { TEST(CliE2E, WriteRejectsInt64Overflow) { std::string err; - EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 3); + EXPECT_EQ(write_one_value("INT64", "99999999999999999999999999", err), 2); EXPECT_NE(err.find("INT64 out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsDoubleOverflow) { std::string err; - EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 3); + EXPECT_EQ(write_one_value("DOUBLE", "1e400", err), 2); EXPECT_NE(err.find("DOUBLE out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericInt64) { std::string err; - EXPECT_EQ(write_one_value("INT64", "12abc", err), 3); + EXPECT_EQ(write_one_value("INT64", "12abc", err), 2); EXPECT_NE(err.find("bad INT64"), std::string::npos) << err; } +TEST(CliE2E, WriteDateRequiresStrictIsoLexicalForm) { + std::string err; + EXPECT_EQ(write_one_value("DATE", "2024-1-1", err), 2); + EXPECT_NE(err.find("want YYYY-MM-DD"), std::string::npos) << err; + EXPECT_EQ(write_one_value("DATE", "2024-02-30", err), 2); + EXPECT_EQ(write_one_value("DATE", "2024-02-29", err), 0) << err; +} + +TEST(CliE2E, WriteAcceptsOneLeadingUtf8Bom) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_bom", ".csv"); + { + std::ofstream o(csv.c_str(), std::ios::binary); + o << "\xEF\xBB\xBFtime,s1\n0,1\n"; + } + std::string output = + tsfile_cli_test::unique_temp_path("tsfile_cli_bom_out", ".tsfile"); + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", output}, + out, err), + 0) + << err.str(); + std::remove(csv.c_str()); + std::remove(output.c_str()); +} + +TEST(CliE2E, WriteRejectsInvalidUtf8AndMisplacedBomWithoutOutput) { + const std::string invalid_utf8 = + std::string("time,s1\n0,") + static_cast(0xC3) + "(\n"; + const std::string misplaced_bom = + "time,s1\n0,\xEF\xBB\xBF" + "1\n"; + const std::string inputs[] = {invalid_utf8, misplaced_bom}; + + for (size_t i = 0; i < 2; ++i) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_bad_utf8", ".csv"); + std::string output = tsfile_cli_test::unique_temp_path( + "tsfile_cli_bad_utf8_out", ".tsfile"); + { + std::ofstream file(csv.c_str(), std::ios::binary); + file.write(inputs[i].data(), + static_cast(inputs[i].size())); + } + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", output}, + out, err), + 2) + << err.str(); + std::ifstream target(output.c_str(), std::ios::binary); + EXPECT_FALSE(target.good()); + std::remove(csv.c_str()); + std::remove(output.c_str()); + } +} + +TEST(CliE2E, WriteRejectsReservedAndControlCharacterNames) { + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t", "--field", "time", + "INT64", "--stdin", "-o", "unused.tsfile"}, + out, err), + 1); + EXPECT_NE(err.str().find("reserved"), std::string::npos) << err.str(); + + out.str(""); + out.clear(); + err.str(""); + err.clear(); + EXPECT_EQ( + tsfile_cli::run_cli({"write", "--table", "bad\nname", "--field", "s1", + "INT64", "--stdin", "-o", "unused.tsfile"}, + out, err), + 1); + EXPECT_NE(err.str().find("control"), std::string::npos) << err.str(); +} + +TEST(CliE2E, WriteFailurePreservesExistingTarget) { + std::string csv = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing", ".csv"); + std::string output = + tsfile_cli_test::unique_temp_path("tsfile_cli_existing_out", ".tsfile"); + { + std::ofstream o(csv.c_str()); + o << "time,s1\n0,1\n"; + } + { + std::ofstream o(output.c_str()); + o << "sentinel"; + } + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", output}, + out, err), + 3); + std::ifstream existing(output.c_str()); + std::stringstream content; + content << existing.rdbuf(); + EXPECT_EQ(content.str(), "sentinel"); + std::remove(csv.c_str()); + std::remove(output.c_str()); +} + TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { // More than one 1024-row batch of ascending rows, then a violating // timestamp. The first batch is already flushed by the time the bad row is @@ -676,10 +1210,10 @@ TEST(CliE2E, WriteRejectsOutOfOrderAcrossBatches) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "t", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); - EXPECT_EQ(code, 3); + EXPECT_EQ(code, 2); EXPECT_NE(err.str().find("strictly increasing"), std::string::npos) << err.str(); EXPECT_FALSE(path_exists(out_path)); @@ -703,27 +1237,30 @@ TEST(CliE2E, WriteStreamsLargeInputRoundTrips) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli({"write", "--table", "big", "--columns", - "s1:INT64:field", "-o", out_path, csv}, + int code = tsfile_cli::run_cli({"write", "--table", "big", "--field", "s1", + "INT64", "-i", csv, "-o", out_path}, out, err); EXPECT_EQ(code, 0) << err.str(); std::ostringstream cout_; std::ostringstream cerr_; - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_); - EXPECT_NE(cout_.str().find("\ts1\t3000"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_); + EXPECT_NE(cout_.str().find("table,big,s1,FIELD,3000,1,3000,0,1,3000,scan"), + std::string::npos) + << cout_.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); } -TEST(CliE2E, HelpWithPositionalFilePrintsUsage) { +TEST(CliE2E, HelpWithPositionalFileIsUsageError) { Fixture f; std::ostringstream out; std::ostringstream err; int code = tsfile_cli::run_cli({"cat", "--help", f.path}, out, err); - EXPECT_EQ(code, 0); - EXPECT_NE(out.str().find("Usage:"), std::string::npos) << out.str(); + EXPECT_EQ(code, 1); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("--help"), std::string::npos) << err.str(); } TEST(CliE2E, StatsRejectsRowOnlyFlag) { @@ -732,8 +1269,7 @@ TEST(CliE2E, StatsRejectsRowOnlyFlag) { std::ostringstream err; int code = tsfile_cli::run_cli({"stats", "--start", "1", f.path}, out, err); EXPECT_EQ(code, 1); - EXPECT_NE(err.str().find("only valid for head/cat/sample"), - std::string::npos) + EXPECT_NE(err.str().find("only valid for head/cat"), std::string::npos) << err.str(); } @@ -762,24 +1298,25 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "id1:STRING:tag,ts1:TIMESTAMP:field,d1:DATE:field,b1:BLOB:field", "-o", - out_path, csv}, + {"write", "--table", "t1", "--tag", "id1", "STRING", "--field", "ts1", + "TIMESTAMP", "--field", "d1", "DATE", "--field", "b1", "BLOB", "-o", + out_path, "-i", csv}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "tsv", out_path}, rout, rerr), + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", out_path}, rout, rerr), 0) << rerr.str(); - // TIMESTAMP prints as raw epoch ms, DATE as YYYY-MM-DD, BLOB as its bytes. + // TIMESTAMP stays a decimal string, DATE uses YYYY-MM-DD, and BLOB uses + // the external 0x-prefixed lowercase hex lexeme. EXPECT_NE(rout.str().find("1700000000000"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-01-15"), std::string::npos) << rout.str(); EXPECT_NE(rout.str().find("2024-12-31"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("hello"), std::string::npos) << rout.str(); - EXPECT_NE(rout.str().find("world"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x68656c6c6f"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("0x776f726c64"), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -787,7 +1324,7 @@ TEST(CliE2E, WriteRoundTripsTimestampDateBlob) { TEST(CliE2E, WriteRejectsBadDate) { std::string err; - EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 3); + EXPECT_EQ(write_one_value("DATE", "not-a-date", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } @@ -803,21 +1340,23 @@ TEST(CliE2E, WriteVerboseEchoesConfig) { std::ostringstream out; std::ostringstream err; - int code = - tsfile_cli::run_cli({"write", "--table", "vt", "--columns", - "s1:INT64:field", "-v", "-o", out_path, csv}, - out, err); + int code = tsfile_cli::run_cli({"write", "--table", "vt", "--field", "s1", + "INT64", "-v", "-i", csv, "-o", out_path}, + out, err); EXPECT_EQ(code, 0) << err.str(); - EXPECT_NE(err.str().find("table=vt"), std::string::npos) << err.str(); - EXPECT_NE(err.str().find("column s1:INT64:field"), std::string::npos) + EXPECT_NE(err.str().find("created model=table object=vt rows=1 output="), + std::string::npos) + << err.str(); + EXPECT_NE(err.str().find("column=s1 category=FIELD data_type=INT64"), + std::string::npos) << err.str(); - EXPECT_NE(err.str().find("wrote 1 rows"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("source=default"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); } -TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { +TEST(CliE2E, WriteRejectsHeaderMatch) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_hm", ".csv"); { @@ -829,14 +1368,12 @@ TEST(CliE2E, WriteHeaderMatchReportsMismatchPosition) { std::ostringstream out; std::ostringstream err; - int code = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "s1:INT64:field", - "--header-match", "-o", out_path, csv}, - out, err); - EXPECT_EQ(code, 3); - EXPECT_NE(err.str().find("header column 2 is 'wrong'"), std::string::npos) - << err.str(); - EXPECT_NE(err.str().find("expected 's1'"), std::string::npos) << err.str(); + int code = + tsfile_cli::run_cli({"write", "--table", "t", "--field", "s1", "INT64", + "--header-match", "-i", csv, "-o", out_path}, + out, err); + EXPECT_EQ(code, 1); + EXPECT_NE(err.str().find("--header-match"), std::string::npos) << err.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); @@ -860,27 +1397,28 @@ TEST(CliE2E, WriteMapsEachColumnToItsOwnValue) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t1", "--columns", - "a_bool:BOOLEAN:field,b_int:INT32:field,c_long:INT64:field," - "d_float:FLOAT:field,e_double:DOUBLE:field,f_str:STRING:field," - "g_ts:TIMESTAMP:field,h_date:DATE:field", - "-o", out_path, csv}, + {"write", "--table", "t1", "--field", "a_bool", "BOOLEAN", + "--field", "b_int", "INT32", "--field", "c_long", "INT64", + "--field", "d_float", "FLOAT", "--field", "e_double", "DOUBLE", + "--field", "f_str", "STRING", "--field", "g_ts", "TIMESTAMP", + "--field", "h_date", "DATE", "-i", csv, "-o", + out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); const std::string& j = rout.str(); EXPECT_NE(j.find("\"a_bool\":true"), std::string::npos) << j; EXPECT_NE(j.find("\"b_int\":42"), std::string::npos) << j; - EXPECT_NE(j.find("\"c_long\":9000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"c_long\":\"9000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"d_float\":1.5"), std::string::npos) << j; EXPECT_NE(j.find("\"e_double\":3.25"), std::string::npos) << j; EXPECT_NE(j.find("\"f_str\":\"hello\""), std::string::npos) << j; - EXPECT_NE(j.find("\"g_ts\":1700000000000"), std::string::npos) << j; + EXPECT_NE(j.find("\"g_ts\":\"1700000000000\""), std::string::npos) << j; EXPECT_NE(j.find("\"h_date\":\"2024-06-15\""), std::string::npos) << j; std::remove(csv.c_str()); @@ -906,26 +1444,29 @@ TEST(CliE2E, WriteMultiTypeAcrossBatchesRoundTrips) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,n:INT64:field,note:TEXT:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "--field", "note", "TEXT", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream cout_; std::ostringstream cerr_; ASSERT_EQ( - tsfile_cli::run_cli({"count", "-f", "tsv", out_path}, cout_, cerr_), 0); - EXPECT_NE(cout_.str().find("\tn\t2500"), std::string::npos) << cout_.str(); + tsfile_cli::run_cli({"count", "-f", "csv", out_path}, cout_, cerr_), 0); + EXPECT_NE(cout_.str().find("table,t,n,FIELD,2500,1,2500,0,0,2499,scan"), + std::string::npos) + << cout_.str(); // Spot-check a row from the last batch keeps n and note paired correctly. std::ostringstream rout; std::ostringstream rerr; ASSERT_EQ(tsfile_cli::run_cli({"cat", "--start", "2400", "--end", "2400", - "-f", "json", out_path}, + "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); - EXPECT_NE(rout.str().find("\"n\":7200"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"n\":\"7200\""), std::string::npos) + << rout.str(); EXPECT_NE(rout.str().find("\"note\":\"row2400\""), std::string::npos) << rout.str(); @@ -950,16 +1491,16 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", - "id:STRING:tag,note:STRING:field", "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "note", + "STRING", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); // JSON escapes the embedded quotes; the comma is preserved verbatim. std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"note\":\"a,b \\\"q\\\" c\""), std::string::npos) @@ -972,20 +1513,20 @@ TEST(CliE2E, WriteRoundTripsQuotedSpecialChars) { TEST(CliE2E, WriteRejectsTimestampOverflow) { std::string err; EXPECT_EQ(write_one_value("TIMESTAMP", "99999999999999999999999999", err), - 3); + 2); EXPECT_NE(err.find("TIMESTAMP out of range"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsNonNumericTimestampColumn) { std::string err; - EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 3); + EXPECT_EQ(write_one_value("TIMESTAMP", "not-a-number", err), 2); EXPECT_NE(err.find("bad TIMESTAMP"), std::string::npos) << err; } TEST(CliE2E, WriteRejectsImpossibleDate) { // Syntactically YYYY-MM-DD but not a real calendar date. std::string err; - EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 3); + EXPECT_EQ(write_one_value("DATE", "2024-13-40", err), 2); EXPECT_NE(err.find("bad DATE"), std::string::npos) << err; } @@ -995,14 +1536,13 @@ TEST(CliE2E, WriteAcceptsDateBoundary) { << err; // leap day } -// An empty cell writes a null, which JSON renders as null (not the type's -// zero). -TEST(CliE2E, WriteEmptyCellBecomesNull) { +// CSV nulls use unquoted \N; quoted empty strings stay distinct from null. +TEST(CliE2E, WriteDistinguishesCsvNullAndEmptyString) { std::string csv = tsfile_cli_test::unique_temp_path("tsfile_cli_null", ".csv"); { std::ofstream o(csv.c_str()); - o << "time,id,n\n0,dev,\n"; // n is empty -> null + o << "time,id,n\n0,dev,\\N\n1,\"\",7\n"; } std::string out_path = tsfile_cli_test::unique_temp_path("tsfile_cli_null_out", ".tsfile"); @@ -1010,17 +1550,18 @@ TEST(CliE2E, WriteEmptyCellBecomesNull) { std::ostringstream wout; std::ostringstream werr; int wc = tsfile_cli::run_cli( - {"write", "--table", "t", "--columns", "id:STRING:tag,n:INT64:field", - "-o", out_path, csv}, + {"write", "--table", "t", "--tag", "id", "STRING", "--field", "n", + "INT64", "-i", csv, "-o", out_path}, wout, werr); ASSERT_EQ(wc, 0) << werr.str(); std::ostringstream rout; std::ostringstream rerr; - ASSERT_EQ(tsfile_cli::run_cli({"cat", "-f", "json", out_path}, rout, rerr), - 0) + ASSERT_EQ( + tsfile_cli::run_cli({"cat", "-f", "ndjson", out_path}, rout, rerr), 0) << rerr.str(); EXPECT_NE(rout.str().find("\"n\":null"), std::string::npos) << rout.str(); + EXPECT_NE(rout.str().find("\"id\":\"\""), std::string::npos) << rout.str(); std::remove(csv.c_str()); std::remove(out_path.c_str()); diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 926772166..844d49464 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/db_common.h" @@ -31,6 +32,21 @@ using tsfile_cli::OutputFormat; using tsfile_cli::ParsedArgs; using tsfile_cli::RowWriter; +namespace { + +class FailingStreamBuf : public std::streambuf { + protected: + std::streamsize xsputn(const char*, std::streamsize) override { return 0; } + int_type overflow(int_type) override { return traits_type::eof(); } +}; + +class FlushFailingStreamBuf : public std::stringbuf { + protected: + int sync() override { return -1; } +}; + +} // namespace + TEST(ErrorCodeMessageTest, KnownCodesMapToReadablePhrases) { EXPECT_STREQ(tsfile_cli::error_code_message(common::E_TABLE_NOT_EXIST), "table does not exist"); @@ -53,11 +69,11 @@ TEST(ErrorCodeMessageTest, UnknownCodeFallsBackToInternalError) { EXPECT_GT(std::string(tsfile_cli::error_code_message(-1)).size(), 0u); } -TEST(ResolveFormatTest, AutoUsesTableOnTtyTsvOtherwise) { +TEST(ResolveFormatTest, AutoAlwaysUsesTable) { EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, true), OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kAuto, false), - OutputFormat::kTsv); + OutputFormat::kTable); EXPECT_EQ(tsfile_cli::resolve_format(ParsedArgs::Format::kJson, true), OutputFormat::kJson); } @@ -119,22 +135,42 @@ TEST(RowWriterTest, NoHeaderSuppressesHeader) { TEST(RowWriterTest, CsvEscapesCells) { std::ostringstream out; - RowWriter w(out, OutputFormat::kCsv, {"name"}, {common::STRING}, false); - w.write({"a,b"}, {false}); + RowWriter w(out, OutputFormat::kCsv, {"name", "note"}, + {common::STRING, common::STRING}, false); + w.write({"a,b", ""}, {false, true}); + w.write({"", ""}, {false, false}); w.finish(); - EXPECT_EQ(out.str(), "name\n\"a,b\"\n"); + EXPECT_EQ(out.str(), "name,note\n\"a,b\",\\N\n\"\",\"\"\n"); } -TEST(RowWriterTest, JsonNumbersUnquotedStringsQuotedNullEmitted) { +TEST(RowWriterTest, JsonQuotesInt64TimestampAndLeavesSmallNumbersBare) { std::ostringstream out; - RowWriter w(out, OutputFormat::kJson, {"time", "name"}, - {common::INT64, common::STRING}, false); - w.write({"5", "dev1"}, {false, false}); - w.write({"6", ""}, {false, true}); + RowWriter w( + out, OutputFormat::kJson, {"time", "small", "ts", "name"}, + {common::INT64, common::INT32, common::TIMESTAMP, common::STRING}, + false); + w.write({"5", "10", "1700000000000", "dev1"}, {false, false, false, false}); + w.write({"6", "11", "1700000000001", ""}, {false, false, false, true}); w.finish(); EXPECT_EQ(out.str(), - "{\"time\":5,\"name\":\"dev1\"}\n" - "{\"time\":6,\"name\":null}\n"); + "{\"time\":\"5\",\"small\":10,\"ts\":\"1700000000000\"," + "\"name\":\"dev1\"}\n" + "{\"time\":\"6\",\"small\":11,\"ts\":\"1700000000001\"," + "\"name\":null}\n"); +} + +TEST(RowWriterTest, BlobCellsUseLowercaseHexLexeme) { + std::ostringstream json; + RowWriter jw(json, OutputFormat::kJson, {"payload"}, {common::BLOB}, false); + jw.write({std::string("A\0z", 3)}, {false}); + jw.finish(); + EXPECT_EQ(json.str(), "{\"payload\":\"0x41007a\"}\n"); + + std::ostringstream csv; + RowWriter cw(csv, OutputFormat::kCsv, {"payload"}, {common::BLOB}, false); + cw.write({"hello"}, {false}); + cw.finish(); + EXPECT_EQ(csv.str(), "payload\n0x68656c6c6f\n"); } TEST(RowWriterTest, TableAlignsColumns) { @@ -149,3 +185,21 @@ TEST(RowWriterTest, TableAlignsColumns) { "s1 INT64\n" "longname BOOLEAN\n"); } + +TEST(RowWriterTest, ReportsStreamWriteFailure) { + FailingStreamBuf buffer; + std::ostream out(&buffer); + RowWriter writer(out, OutputFormat::kCsv, {"name"}, {common::STRING}, + false); + EXPECT_FALSE(writer.write({"value"}, {false})); + EXPECT_FALSE(writer.finish()); +} + +TEST(RowWriterTest, ReportsFlushFailure) { + FlushFailingStreamBuf buffer; + std::ostream out(&buffer); + RowWriter writer(out, OutputFormat::kCsv, {"name"}, {common::STRING}, + false); + ASSERT_TRUE(writer.write({"value"}, {false})); + EXPECT_FALSE(writer.finish()); +} diff --git a/cpp/test/tools/statistics_test.cc b/cpp/test/tools/statistics_test.cc index a151fdc3c..c6c7d8738 100644 --- a/cpp/test/tools/statistics_test.cc +++ b/cpp/test/tools/statistics_test.cc @@ -23,7 +23,18 @@ #include "common/statistic.h" -TEST(StatisticsTest, Int64StatisticCellsContainValueSummaries) { +TEST(StatisticsTest, Int32StatisticSupportsAllValueSummaries) { + storage::Int32Statistic st; + st.update(1, static_cast(10)); + st.update(3, static_cast(30)); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.values, + std::vector({"10", "30", "10", "30", "40"})); + EXPECT_EQ(cells.is_null, + std::vector({false, false, false, false, false})); +} + +TEST(StatisticsTest, Int64StatisticLeavesSumNull) { storage::Int64Statistic st; st.update(1, static_cast(10)); st.update(3, static_cast(30)); @@ -32,9 +43,27 @@ TEST(StatisticsTest, Int64StatisticCellsContainValueSummaries) { EXPECT_EQ(cells.values[1], "30"); EXPECT_EQ(cells.values[2], "10"); EXPECT_EQ(cells.values[3], "30"); - EXPECT_EQ(cells.values[4], "40"); + EXPECT_TRUE(cells.values[4].empty()); EXPECT_EQ(cells.is_null, - std::vector({false, false, false, false, false})); + std::vector({false, false, false, false, true})); +} + +TEST(StatisticsTest, DateAndTimestampStatisticsLeaveSumNull) { + storage::DateStatistic date; + date.update(1, static_cast(19700101)); + tsfile_cli::StatisticCells date_cells = + tsfile_cli::statistic_value_cells(&date); + EXPECT_EQ(date_cells.values[0], "1970-01-01"); + EXPECT_EQ(date_cells.values[3], "1970-01-01"); + EXPECT_TRUE(date_cells.is_null[4]); + + storage::TimestampStatistics timestamp; + timestamp.update(1, static_cast(1700000000000)); + tsfile_cli::StatisticCells timestamp_cells = + tsfile_cli::statistic_value_cells(×tamp); + EXPECT_EQ(timestamp_cells.values[0], "1700000000000"); + EXPECT_EQ(timestamp_cells.values[3], "1700000000000"); + EXPECT_TRUE(timestamp_cells.is_null[4]); } TEST(StatisticsTest, BooleanStatisticLeavesMinMaxNull) { @@ -48,3 +77,53 @@ TEST(StatisticsTest, BooleanStatisticLeavesMinMaxNull) { EXPECT_EQ(cells.values[3], "false"); EXPECT_EQ(cells.values[4], "1"); } + +TEST(StatisticsTest, DoubleStatisticPreservesRoundTripPrecision) { + storage::DoubleStatistic st; + st.update(1, 1.2345678901234567); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.values[0], "1.2345678901234567"); + EXPECT_EQ(cells.values[4], "1.2345678901234567"); +} + +TEST(StatisticsTest, FloatStatisticSupportsAllValueSummaries) { + storage::FloatStatistic st; + st.update(1, 1.25F); + st.update(2, 2.5F); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.values, + std::vector({"1.25", "2.5", "1.25", "2.5", "3.75"})); + EXPECT_EQ(cells.is_null, + std::vector({false, false, false, false, false})); +} + +TEST(StatisticsTest, StringStatisticOmitsOnlySum) { + storage::StringStatistic st; + st.update(1, common::String("beta")); + st.update(2, common::String("alpha")); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.values[0], "alpha"); + EXPECT_EQ(cells.values[1], "beta"); + EXPECT_EQ(cells.values[2], "beta"); + EXPECT_EQ(cells.values[3], "alpha"); + EXPECT_EQ(cells.is_null, + std::vector({false, false, false, false, true})); +} + +TEST(StatisticsTest, TextStatisticOnlyProvidesFirstAndLast) { + storage::TextStatistic st; + st.update(1, common::String("first")); + st.update(2, common::String("last")); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.values[2], "first"); + EXPECT_EQ(cells.values[3], "last"); + EXPECT_EQ(cells.is_null, + std::vector({true, true, false, false, true})); +} + +TEST(StatisticsTest, BlobStatisticProvidesNoValueSummaries) { + storage::BlobStatistic st; + st.update(1, common::String("blob")); + tsfile_cli::StatisticCells cells = tsfile_cli::statistic_value_cells(&st); + EXPECT_EQ(cells.is_null, std::vector({true, true, true, true, true})); +} diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 4448feab2..00bc20ea0 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -36,8 +36,21 @@ target_include_directories(tsfile_cli_obj PUBLIC # can compile these sources before the headers exist. add_dependencies(tsfile_cli_obj tsfile) +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE TSFILE_CLI_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if (NOT TSFILE_CLI_GIT_COMMIT) + set(TSFILE_CLI_GIT_COMMIT "unknown") +endif () +string(TIMESTAMP TSFILE_CLI_BUILD_TIME "%Y-%m-%dT%H:%M:%SZ" UTC) + target_compile_definitions(tsfile_cli_obj PRIVATE - TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}") + TSFILE_CLI_VERSION="${TsFile_CPP_VERSION}" + TSFILE_CLI_COMMIT="${TSFILE_CLI_GIT_COMMIT}" + TSFILE_CLI_BUILT="${TSFILE_CLI_BUILD_TIME}") add_executable(tsfile_cli tools_main.cc $) target_include_directories(tsfile_cli PRIVATE ${CMAKE_SOURCE_DIR}/tools) diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 60f7acc07..37f704896 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -25,7 +25,7 @@ importing Apache TsFile (`.tsfile`) files from the shell — the TsFile analogue of `parquet-cli` / `pqrs`. Read commands print data to **stdout** and diagnostics to **stderr**, so they compose with `awk`, `jq`, `sort`, and friends; the `write` command -imports CSV/TSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` +imports CSV into a new `.tsfile`. It is built on the public `storage::TsFileReader` and `storage::TsFileTableWriter` APIs and does not modify the storage engine. ## Building from source @@ -89,14 +89,15 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, | Command | Description | |---|---| -| `ls` | List devices (tree model) or tables (table model), one name per line | -| `schema` | Per-series `target, measurement, datatype, encoding, compression` | -| `meta` | File summary: model, device/table/series counts, time range, file size | -| `stats` | Per-series `count, start_time, end_time, min, max, first, last, sum` | -| `count` | Per-series row counts plus a `total` row (from statistics, no page scan) | +| `ls` | List selected-model objects as `model, object` rows | +| `schema` | List schema rows for devices or tables | +| `meta` | File summary: `size_bytes`, `format_version`, and `model` | +| `stats` | FIELD statistics with counts, null counts, time range, values, and source | +| `count` | Object/column counts; no synthetic summary row | +| `sketch` | Print the physical file sketch, optionally to `-o` | | `head` | First N rows (default 10; use `-n`) | | `cat` | All matching rows, streamed (`table` format buffers to align columns) | -| `sample` | Reproducible reservoir sample (default 10; `-n`, `--seed`) | +| `export` | Export one object to `-o`, or multiple objects to `--output-dir`, using `--type` | The metadata commands (`ls` / `schema` / `meta` / `stats` / `count`) answer most questions without decoding data pages. @@ -105,73 +106,82 @@ Shared options: | Option | Meaning | |---|---| -| `-f, --format csv\|tsv\|json\|table` | Output format; defaults to `table` on a TTY, `tsv` when piped | +| `-f, --format table\|ndjson\|csv` | Output format; defaults to `table` | | `-d, --device ` / `-t, --table ` | Scope to one device / table (mutually exclusive) | -| `-m, --measurements a,b,c` | Column projection (`schema`, `stats`, `count`, `head`, `cat`, `sample`) | -| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`; `--offset` not valid for `sample`) | -| `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `sample`) | -| `--seed N` | Reproducible sampling seed (`sample` only) | -| `--tag-filter C OP V` / `--tag-between C L U` / `--tag-not-between C L U` | Table TAG predicate for `head`, `cat`, `sample`; `OP` is `eq`, `neq`, `lt`, `lteq`, `gt`, `gteq`, `regexp`, or `not-regexp` | -| `--no-header` | Omit the header row | -| `--model tree\|table` | Force the model (otherwise auto-detected) | - -`json` output is NDJSON (one object per line; numbers/booleans bare, other values quoted, -nulls as `null`; non-finite floats — NaN/Inf — become `null`). CSV output follows RFC 4180. -Timestamps are raw epoch milliseconds. The `table` format buffers all rows in memory to -align columns, so prefer `csv`/`tsv`/`json` when dumping large files. +| `-m, --measurements ` | Column projection; repeat once per column. For `stats`, only FIELD columns are valid | +| `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`, `export`) | +| `--start