diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 9b4d8aef4..49f5fd9f8 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -399,16 +399,14 @@ int TsFileIOReader::get_device_timeseries_meta_without_chunk_meta( } std::shared_ptr meta_index_entry; int64_t end_offset; - std::vector, int64_t>> - meta_index_entry_list; + // Keep the shared time index in filtered metadata so table statistics + // can derive null counts from the footer. if (RET_FAIL(load_device_index_entry( std::make_shared(device_id), meta_index_entry, end_offset))) { - } else if (RET_FAIL(load_all_measurement_index_entry( - meta_index_entry->get_offset(), end_offset, pa, - meta_index_entry_list))) { - } else if (RET_FAIL(do_load_all_timeseries_index(meta_index_entry_list, pa, - timeseries_indexs))) { + } else if (RET_FAIL(get_device_timeseries_meta_by_offset( + meta_index_entry->get_offset(), end_offset, + timeseries_indexs, pa))) { } return ret; } diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 314d69c6d..a9ae1f84a 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -826,6 +826,13 @@ TEST_F(TsFileReaderTest, GetTimeseriesMetadataTableModelTypeAndDeviceFilter) { auto selected_list = selected_meta.begin()->second; std::unordered_map type_by_measurement; for (const auto& index : selected_list) { + auto* aligned = + dynamic_cast(index.get()); + ASSERT_NE(aligned, nullptr) + << "filtered metadata must preserve the shared time index"; + ASSERT_NE(aligned->time_ts_idx_, nullptr); + ASSERT_NE(aligned->time_ts_idx_->get_statistic(), nullptr); + EXPECT_EQ(aligned->time_ts_idx_->get_statistic()->get_count(), 5); type_by_measurement[index->get_measurement_name().to_std_string()] = index->get_data_type(); } diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index 083000991..fae33226d 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -165,11 +165,55 @@ TEST(CliE2E, StatsReportsCountAndTimeRange) { std::string::npos) << out.str(); EXPECT_NE(out.str().find("table,table1,id1_field_1,id2_field_2,s1,INT64," - "5,\\N,0,4,0,40,0,40,\\N,statistics"), + "5,0,0,4,0,40,0,40,\\N,statistics"), std::string::npos) << out.str(); } +TEST(CliE2E, StatsReportsNumericNullCountForTableFields) { + // Filtered metadata must preserve the shared time statistic so the + // footer's row count and each field's non-null count determine null_count. + const std::string csv_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_null_count", ".csv"); + const std::string out_path = + tsfile_cli_test::unique_temp_path("tsfile_cli_null_count", ".tsfile"); + { + std::ofstream csv(csv_path.c_str(), std::ios::binary); + ASSERT_TRUE(csv.good()); + csv << "time,site,temp,note\n" + "1000,beijing,20,n0\n" + "2000,shanghai,21,n1\n" + "3000,beijing,22,\\N\n" + "4000,shanghai,23,n3\n"; + } + std::ostringstream wout, werr; + ASSERT_EQ( + tsfile_cli::run_cli({"write", "--table", "sensors", "--tag", "site", + "STRING", "--field", "temp", "FLOAT", "--field", + "note", "TEXT", "-i", csv_path, "-o", out_path}, + wout, werr), + 0) + << werr.str(); + + std::ostringstream out, err; + ASSERT_EQ(tsfile_cli::run_cli({"stats", "-t", "sensors", "-m", "note", "-f", + "csv", out_path}, + out, err), + 0) + << err.str(); + EXPECT_TRUE(err.str().empty()) << err.str(); + EXPECT_EQ(out.str(), + "model,object,tag.site,field,data_type,non_null_count,null_count," + "min_time,max_time,min,max,first,last,sum,stats_source\n" + "table,sensors,beijing,note,TEXT,1,1,1000,1000,\\N,\\N,n0,n0," + "\\N,statistics\n" + "table,sensors,shanghai,note,TEXT,2,0,2000,4000,\\N,\\N,n1,n3," + "\\N,statistics\n"); + + std::remove(csv_path.c_str()); + std::remove(out_path.c_str()); +} + TEST(CliE2E, HeadProjectsAndLimits) { Fixture f; std::ostringstream out; @@ -692,7 +736,7 @@ TEST(CliE2E, MetadataTableFilterIsCaseInsensitive) { stats_out, stats_err), 0); EXPECT_NE(stats_out.str().find("table,table1,id1_field_1,id2_field_2,s1," - "INT64,5,\\N,0,4,0,40,0,40,\\N,statistics"), + "INT64,5,0,0,4,0,40,0,40,\\N,statistics"), std::string::npos) << stats_out.str(); } diff --git a/cpp/test/tools/golden/table_stats_csv.txt b/cpp/test/tools/golden/table_stats_csv.txt index d25d0085f..6c539d5dc 100644 --- a/cpp/test/tools/golden/table_stats_csv.txt +++ b/cpp/test/tools/golden/table_stats_csv.txt @@ -1,5 +1,5 @@ model,object,tag.site,tag.rack,tag.sensor,field,data_type,non_null_count,null_count,min_time,max_time,min,max,first,last,sum,stats_source -table,table1,\N,\N,\N,value,INT64,20,\N,0,19,0,19,0,19,\N,statistics +table,table1,\N,\N,\N,value,INT64,20,0,0,19,0,19,0,19,\N,statistics table,table2,\N,rack0,sensor0,reading,DOUBLE,0,1,0,0,\N,\N,\N,\N,\N,scan table,table2,"",rack1,sensor1,reading,DOUBLE,1,0,1,1,1.25,1.25,1.25,1.25,1.25,statistics table,table2,null,rack2,sensor2,reading,DOUBLE,1,0,2,2,2.5,2.5,2.5,2.5,2.5,statistics diff --git a/cpp/test/tools/golden/table_stats_ndjson.txt b/cpp/test/tools/golden/table_stats_ndjson.txt index 052a7b3ae..82f165d19 100644 --- a/cpp/test/tools/golden/table_stats_ndjson.txt +++ b/cpp/test/tools/golden/table_stats_ndjson.txt @@ -1,4 +1,4 @@ -{"model":"table","object":"table1","tag.site":null,"tag.rack":null,"tag.sensor":null,"field":"value","data_type":"INT64","non_null_count":"20","null_count":null,"min_time":"0","max_time":"19","min":"0","max":"19","first":"0","last":"19","sum":null,"stats_source":"statistics"} +{"model":"table","object":"table1","tag.site":null,"tag.rack":null,"tag.sensor":null,"field":"value","data_type":"INT64","non_null_count":"20","null_count":"0","min_time":"0","max_time":"19","min":"0","max":"19","first":"0","last":"19","sum":null,"stats_source":"statistics"} {"model":"table","object":"table2","tag.site":null,"tag.rack":"rack0","tag.sensor":"sensor0","field":"reading","data_type":"DOUBLE","non_null_count":"0","null_count":"1","min_time":"0","max_time":"0","min":null,"max":null,"first":null,"last":null,"sum":null,"stats_source":"scan"} {"model":"table","object":"table2","tag.site":"","tag.rack":"rack1","tag.sensor":"sensor1","field":"reading","data_type":"DOUBLE","non_null_count":"1","null_count":"0","min_time":"1","max_time":"1","min":1.25,"max":1.25,"first":1.25,"last":1.25,"sum":1.25,"stats_source":"statistics"} {"model":"table","object":"table2","tag.site":"null","tag.rack":"rack2","tag.sensor":"sensor2","field":"reading","data_type":"DOUBLE","non_null_count":"1","null_count":"0","min_time":"2","max_time":"2","min":2.5,"max":2.5,"first":2.5,"last":2.5,"sum":2.5,"stats_source":"statistics"} diff --git a/cpp/test/tools/golden/table_stats_table.txt b/cpp/test/tools/golden/table_stats_table.txt index e0d6d4207..e5eb67371 100644 --- a/cpp/test/tools/golden/table_stats_table.txt +++ b/cpp/test/tools/golden/table_stats_table.txt @@ -1,5 +1,5 @@ model object tag.site tag.rack tag.sensor field data_type non_null_count null_count min_time max_time min max first last sum stats_source -table table1 value INT64 20 0 19 0 19 0 19 statistics +table table1 value INT64 20 0 0 19 0 19 0 19 statistics table table2 rack0 sensor0 reading DOUBLE 0 1 0 0 scan table table2 rack1 sensor1 reading DOUBLE 1 0 1 1 1.25 1.25 1.25 1.25 1.25 statistics table table2 null rack2 sensor2 reading DOUBLE 1 0 2 2 2.5 2.5 2.5 2.5 2.5 statistics