From 5ebd441c88b0e57a5d131e219c44af29d78e4fab Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Sun, 19 Jan 2025 18:02:01 -0600
Subject: [PATCH 1/9] add index cache layer

---
 src/metric_engine/src/data/mod.rs    |  1 +
 src/metric_engine/src/index/cache.rs | 49 +++++++++++++++++
 src/metric_engine/src/index/mod.rs   | 78 ++++++++++++++++++++++++++--
 src/metric_engine/src/lib.rs         |  2 +
 src/metric_engine/src/metric/mod.rs  | 15 ++++--
 src/metric_engine/src/types.rs       | 44 +++++++++++++++-
 6 files changed, 180 insertions(+), 9 deletions(-)
 create mode 100644 src/metric_engine/src/index/cache.rs

diff --git a/src/metric_engine/src/data/mod.rs b/src/metric_engine/src/data/mod.rs
index 96d1a9d78f..ae6a9b89d7 100644
--- a/src/metric_engine/src/data/mod.rs
+++ b/src/metric_engine/src/data/mod.rs
@@ -35,6 +35,7 @@ impl SampleManager {
     /// Populate series ids from labels.
     /// It will also build inverted index for labels.
     pub async fn persist(&self, _samples: Vec<Sample>) -> Result<()> {
+        // 1. just write to TM storage directly
         todo!()
     }
 }
diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
new file mode 100644
index 0000000000..48f6cc3799
--- /dev/null
+++ b/src/metric_engine/src/index/cache.rs
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::{HashMap, HashSet};
+
+use crate::types::{
+    FieldName, FieldType, MetricId, MetricName, SegmentDuration, SeriesId, SeriesKey, TagName,
+    TagValue,
+};
+
+type MetricsCache = HashMap<SegmentDuration, HashMap<MetricName, (FieldName, FieldType)>>;
+type SeriesCache = HashMap<SegmentDuration, HashMap<SeriesId, SeriesKey>>;
+type TagIndexCache =
+    HashMap<SegmentDuration, HashMap<TagName, HashMap<TagValue, HashSet<(SeriesId, MetricId)>>>>;
+
+#[derive(Default)]
+pub struct CacheManager {
+    metrics: MetricsCache,
+    series: SeriesCache,
+    tag_index: TagIndexCache,
+}
+
+impl CacheManager {
+    pub fn update_metric() -> Option<()> {
+        todo!()
+    }
+
+    pub fn update_series() -> Option<()> {
+        todo!()
+    }
+
+    pub fn update_tag_index() -> Option<()> {
+        todo!()
+    }
+}
diff --git a/src/metric_engine/src/index/mod.rs b/src/metric_engine/src/index/mod.rs
index 4d98db4f5b..2f1cd35d71 100644
--- a/src/metric_engine/src/index/mod.rs
+++ b/src/metric_engine/src/index/mod.rs
@@ -15,11 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
+mod cache;
 use std::sync::Arc;
 
+use cache::CacheManager;
 use horaedb_storage::storage::TimeMergeStorageRef;
 
-use crate::{types::Sample, Result};
+use crate::{
+    types::{
+        hash, FieldName, FieldType, MetricId, MetricName, Sample, SegmentDuration, SeriesId,
+        SeriesKey, TagName, TagValue,
+    },
+    Result,
+};
 
 pub struct IndexManager {
     inner: Arc<Inner>,
@@ -28,17 +36,81 @@ pub struct IndexManager {
 impl IndexManager {
     pub fn new(storage: TimeMergeStorageRef) -> Self {
         Self {
-            inner: Arc::new(Inner { storage }),
+            inner: Arc::new(Inner {
+                storage,
+                cache: CacheManager::default(),
+            }),
         }
     }
 
     /// Populate series ids from labels.
     /// It will also build inverted index for labels.
-    pub async fn populate_series_ids(&self, _samples: &mut [Sample]) -> Result<()> {
+    pub async fn populate_series_ids(&self, samples: &mut [Sample]) -> Result<()> {
+        // 1. create metric id and series id
+        let metric_ids = samples
+            .iter()
+            .map(|s| MetricId(hash(s.name.as_slice())))
+            .collect::<Vec<_>>();
+
+        let series_keys = samples
+            .iter()
+            .map(|s| SeriesKey::new(s.name.as_slice(), s.lables.as_slice()))
+            .collect::<Vec<_>>();
+        let series_ids = series_keys
+            .iter()
+            .map(|e| SeriesId(hash(e.make_bytes().as_slice())))
+            .collect::<Vec<_>>();
+
+        samples.iter_mut().enumerate().for_each(|(i, sample)| {
+            sample.name_id = Some(metric_ids[i]);
+            sample.series_id = Some(series_ids[i]);
+        });
+        // 2. cache metrics
+        samples
+            .iter()
+            .for_each(|s| self.inner.create_metrics(s.name.as_slice()));
+
+        // 3. cache series
+        series_ids
+            .iter()
+            .zip(series_keys.iter())
+            .for_each(|(id, key)| self.inner.create_series(id, key));
+
+        // 4. cache tag index
+        series_ids
+            .iter()
+            .zip(series_keys.iter())
+            .zip(metric_ids.iter())
+            .for_each(|((series_id, series_key), metric_id)| {
+                self.inner
+                    .create_tag_index(series_id, series_key, metric_id)
+            });
+
+        // 5. write all
         todo!()
     }
 }
 
 struct Inner {
     storage: TimeMergeStorageRef,
+    cache: CacheManager,
+}
+
+impl Inner {
+    pub fn create_metrics(&self, name: &[u8]) {
+        todo!()
+    }
+
+    pub fn create_series(&self, id: &SeriesId, key: &SeriesKey) {
+        todo!()
+    }
+
+    pub fn create_tag_index(
+        &self,
+        series_id: &SeriesId,
+        series_key: &SeriesKey,
+        metric_id: &MetricId,
+    ) {
+        todo!()
+    }
 }
diff --git a/src/metric_engine/src/lib.rs b/src/metric_engine/src/lib.rs
index 2c64fe3361..e705f00276 100644
--- a/src/metric_engine/src/lib.rs
+++ b/src/metric_engine/src/lib.rs
@@ -17,6 +17,8 @@
 
 //! Metric Engine entry point.
 
+mod data;
+mod index;
 mod metric;
 mod types;
 
diff --git a/src/metric_engine/src/metric/mod.rs b/src/metric_engine/src/metric/mod.rs
index bdd1d6623c..69f8e77baf 100644
--- a/src/metric_engine/src/metric/mod.rs
+++ b/src/metric_engine/src/metric/mod.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 
 use horaedb_storage::storage::TimeMergeStorageRef;
 
-use crate::{types::Sample, Result};
+use crate::{index::IndexManager, types::Sample, Result};
 
 pub struct MetricManager {
     inner: Arc<Inner>,
@@ -28,23 +28,30 @@ pub struct MetricManager {
 impl MetricManager {
     pub fn new(storage: TimeMergeStorageRef) -> Self {
         Self {
-            inner: Arc::new(Inner { storage }),
+            inner: Arc::new(Inner {
+                storage: storage.clone(),
+                index: IndexManager::new(storage),
+            }),
         }
     }
 
     /// Populate metric ids from names.
     /// If a name does not exist, it will be created on demand.
     pub async fn populate_metric_ids(&self, _samples: &mut [Sample]) -> Result<()> {
+        // 1. call index manager to create index
+
+        // 2. call data manager to write samples
         todo!()
     }
 }
 
 struct Inner {
     storage: TimeMergeStorageRef,
+    index: IndexManager,
 }
 
 impl Inner {
-    async fn populate_metric_ids(&self, _samples: &mut [Sample]) -> Result<()> {
-        todo!()
+    async fn populate_metric_ids(&self, samples: &mut [Sample]) -> Result<()> {
+        self.index.populate_series_ids(samples).await
     }
 }
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index 6203bc174d..9f6f1e35ee 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -14,10 +14,20 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+use std::{collections::BTreeSet, io::Write, time::Duration};
 
-pub struct MetricId(u64);
-pub struct SeriesId(u64);
+#[derive(Copy, Clone)]
+pub struct MetricId(pub u64);
+#[derive(Copy, Clone)]
+pub struct SeriesId(pub u64);
+pub type SegmentDuration = Duration;
+pub type MetricName = Vec<u8>;
+pub type FieldName = Vec<u8>;
+pub type FieldType = usize;
+pub type TagName = Vec<u8>;
+pub type TagValue = Vec<u8>;
 
+#[derive(PartialEq, PartialOrd, Eq, Ord, Clone)]
 pub struct Label {
     pub name: Vec<u8>,
     pub value: Vec<u8>,
@@ -36,6 +46,36 @@ pub struct Sample {
     pub series_id: Option<SeriesId>,
 }
 
+pub struct SeriesKey(Vec<Label>);
+
+impl SeriesKey {
+    pub fn new(metric_name: &[u8], lables: &[Label]) -> Self {
+        let mut set: BTreeSet<Label> = BTreeSet::new();
+        lables.iter().for_each(|item| {
+            set.insert(item.clone());
+        });
+        set.insert(Label {
+            name: String::from("__name__").into(),
+            value: metric_name.to_vec(),
+        });
+
+        Self(set.into_iter().collect::<Vec<Label>>())
+    }
+
+    pub fn make_bytes(&self) -> Vec<u8> {
+        let mut series_bytes: Vec<u8> = Vec::new();
+        self.0.iter().for_each(|item| {
+            series_bytes
+                .write_all(item.name.as_slice())
+                .expect("can write");
+            series_bytes
+                .write_all(item.value.as_slice())
+                .expect("can write");
+        });
+        series_bytes
+    }
+}
+
 pub fn hash(buf: &[u8]) -> u64 {
     seahash::hash(buf)
 }

From 37826144b77d42d40551ea442cc1f01ebf5aaf5f Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Wed, 22 Jan 2025 22:18:41 -0600
Subject: [PATCH 2/9] add index cache layer

---
 Cargo.lock                           |   4 +
 Cargo.toml                           |   1 +
 src/metric_engine/Cargo.toml         |   4 +
 src/metric_engine/src/index/cache.rs | 812 ++++++++++++++++++++++++++-
 src/metric_engine/src/index/mod.rs   |  21 +-
 src/metric_engine/src/metric/mod.rs  |   4 +-
 src/metric_engine/src/types.rs       |  97 +++-
 7 files changed, 899 insertions(+), 44 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f92b955d32..ca5e02012a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2154,7 +2154,11 @@ name = "metric_engine"
 version = "2.2.0-alpha"
 dependencies = [
  "anyhow",
+ "arrow",
  "common",
+ "dashmap",
+ "futures",
+ "object_store",
  "seahash",
  "storage",
  "temp-dir",
diff --git a/Cargo.toml b/Cargo.toml
index 84ae73c6e3..1a55200b4c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -44,6 +44,7 @@ common = { path = "src/common" }
 thiserror = "1"
 bytes = "1"
 byteorder = "1"
+dashmap = { version = "6.1.0" }
 datafusion = "43"
 parquet = { version = "53" }
 object_store = { version = "0.11" }
diff --git a/src/metric_engine/Cargo.toml b/src/metric_engine/Cargo.toml
index ceaf134241..0ecf1492e9 100644
--- a/src/metric_engine/Cargo.toml
+++ b/src/metric_engine/Cargo.toml
@@ -27,8 +27,12 @@ description.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
+arrow = { workspace = true }
 common = { workspace = true }
+dashmap = { workspace = true }
+futures = { workspace = true }
 horaedb_storage = { workspace = true }
+object_store = { workspace = true }
 seahash = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index 48f6cc3799..480d28686b 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -1,3 +1,4 @@
+use core::num;
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -14,36 +15,819 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+use std::{
+    collections::{HashMap, HashSet},
+    path::Path,
+    sync::{Arc, RwLock},
+    time::Duration,
+};
 
-use std::collections::{HashMap, HashSet};
+use arrow::{
+    array::{
+        Array, ArrayRef, BinaryArray, BinaryBuilder, Int32BufferBuilder, ListArray, ListBuilder,
+        UInt64Array, UInt64Builder, UInt8Array,
+    },
+    buffer::OffsetBuffer,
+    datatypes::{BinaryType, DataType, Field, Schema, ToByteSlice},
+    record_batch::RecordBatch,
+};
+use dashmap::DashMap;
+use futures::{Stream, StreamExt};
+use horaedb_storage::{
+    config::StorageConfig,
+    storage::{
+        CloudObjectStorage, ScanRequest, StorageRuntimes, TimeMergeStorage, TimeMergeStorageRef,
+        WriteRequest,
+    },
+    types::{ObjectStoreRef, TimeRange, Timestamp},
+};
+use tokio::{
+    runtime::Runtime,
+    sync::mpsc::{self, Receiver, Sender},
+};
 
 use crate::types::{
-    FieldName, FieldType, MetricId, MetricName, SegmentDuration, SeriesId, SeriesKey, TagName,
-    TagValue,
+    default_field_name, default_field_type, hash, FieldName, FieldType, Label, MetricId,
+    MetricName, SegmentDuration, SeriesId, SeriesKey, TagName, TagNames, TagValue, TagValues,
 };
 
-type MetricsCache = HashMap<SegmentDuration, HashMap<MetricName, (FieldName, FieldType)>>;
-type SeriesCache = HashMap<SegmentDuration, HashMap<SeriesId, SeriesKey>>;
-type TagIndexCache =
-    HashMap<SegmentDuration, HashMap<TagName, HashMap<TagValue, HashSet<(SeriesId, MetricId)>>>>;
+struct MetricsCache {
+    cache: DashMap<SegmentDuration, RwLock<HashMap<MetricName, (FieldName, FieldType)>>>,
+    pub storage: TimeMergeStorageRef,
+    sender: Sender<Task>,
+}
+struct SeriesCache {
+    cache: DashMap<SegmentDuration, RwLock<HashMap<SeriesId, SeriesKey>>>,
+    pub storage: TimeMergeStorageRef,
+    sender: Sender<Task>,
+}
+
+#[derive(PartialEq, Eq, Hash, Debug)]
+struct SegmentSeries {
+    segment: SegmentDuration,
+    series_id: SeriesId,
+}
+
+struct TagIndexCache {
+    cache: DashMap<
+        SegmentDuration,
+        RwLock<HashMap<TagName, HashMap<TagValue, HashMap<MetricId, HashSet<SeriesId>>>>>,
+    >,
+    write_guard: RwLock<HashSet<SegmentSeries>>,
+    pub storage: TimeMergeStorageRef,
+    sender: Sender<Task>,
+}
+
+impl MetricsCache {
+    fn new(storage: TimeMergeStorageRef, sender: Sender<Task>) -> Self {
+        Self {
+            cache: DashMap::new(),
+            storage,
+            sender,
+        }
+    }
+
+    async fn load_from_storage(&mut self) {
+        let mut result_stream = self
+            .storage
+            .scan(ScanRequest {
+                range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                predicate: vec![],
+                projections: None,
+            })
+            .await
+            .unwrap();
+        while let Some(item) = result_stream.next().await {
+            let batch = item.unwrap();
+            for index in 0..batch.num_rows() {
+                let metric_name = batch
+                    .column_by_name("metric_name")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .unwrap()
+                    .value(index);
+
+                let field_name = batch
+                    .column_by_name("field_name")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .unwrap()
+                    .value(index);
+
+                let field_type = batch
+                    .column_by_name("field_type")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt8Array>()
+                    .unwrap()
+                    .value(index);
+
+                // TODO: why not put in cache?
+                let _filed_id = batch
+                    .column_by_name("field_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+
+                // TODO: why not put in cache?
+                let _metric_id = batch
+                    .column_by_name("metric_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+
+                self.update(metric_name, field_name, field_type, None).await;
+            }
+        }
+    }
+
+    fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("metric_name", DataType::Binary, true),
+            Field::new("metric_id", DataType::UInt64, true),
+            Field::new("field_name", DataType::Binary, true),
+            Field::new("field_id", DataType::UInt64, true),
+            Field::new("field_type", DataType::UInt8, true),
+        ]))
+    }
+
+    async fn update(&self, name: &[u8], field_name: &[u8], field_type: u8, notify: Option<()>) {
+        let current = SegmentDuration::current();
+        let task = if self.cache.contains_key(&current)
+            && self
+                .cache
+                .get(&current)
+                .unwrap()
+                .read()
+                .unwrap()
+                .contains_key(name)
+        {
+            None
+        } else {
+            let result = self
+                .cache
+                .entry(current)
+                .or_default()
+                .write()
+                .unwrap()
+                .insert(name.to_vec(), (field_name.to_vec(), field_type));
+
+            match result {
+                Some(_) => None,
+                None => Some(Task::Metric(name.to_vec())),
+            }
+        };
+
+        if let (Some(task), Some(_)) = (task, notify) {
+            self.sender.send(task).await.unwrap();
+        }
+    }
+}
+
+impl SeriesCache {
+    fn new(storage: TimeMergeStorageRef, sender: Sender<Task>) -> Self {
+        Self {
+            cache: DashMap::new(),
+            storage,
+            sender,
+        }
+    }
+
+    async fn load_from_storage(&mut self) {
+        let mut result_stream = self
+            .storage
+            .scan(ScanRequest {
+                range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                predicate: vec![],
+                projections: None,
+            })
+            .await
+            .unwrap();
+        while let Some(item) = result_stream.next().await {
+            let batch = item.unwrap();
+            for index in 0..batch.num_rows() {
+                let metric_id = batch
+                    .column_by_name("metric_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+
+                let series_id = batch
+                    .column_by_name("series_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+
+                let tag_name_array = batch
+                    .column_by_name("tag_names")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<ListArray>()
+                    .unwrap()
+                    .value(index);
+                let tag_names = tag_name_array
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .expect("List elements should be BinaryArray");
+
+                let names = tag_names
+                    .iter()
+                    .map(|item| item.unwrap().to_vec())
+                    .collect::<Vec<_>>();
+
+                let tag_value_array = batch
+                    .column_by_name("tag_values")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<ListArray>()
+                    .unwrap()
+                    .value(index);
+                let tag_values = tag_value_array
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .expect("List elements should be BinaryArray");
+
+                let values = tag_values
+                    .iter()
+                    .map(|item| item.unwrap().to_vec())
+                    .collect::<Vec<_>>();
+
+                let labels = names
+                    .into_iter()
+                    .zip(values.into_iter())
+                    .map(|(name, value)| Label { name, value })
+                    .collect::<Vec<_>>();
+                let key = SeriesKey::new(None, labels.as_slice());
+                self.update(&SeriesId(series_id), &key, &MetricId(metric_id), None)
+                    .await;
+            }
+        }
+    }
+
+    pub fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("metric_id", DataType::UInt64, true),
+            Field::new("series_id", DataType::UInt64, true),
+            Field::new(
+                "tag_names",
+                DataType::List(Arc::new(Field::new("item", DataType::Binary, true))),
+                true,
+            ),
+            Field::new(
+                "tag_values",
+                DataType::List(Arc::new(Field::new("item", DataType::Binary, true))),
+                true,
+            ),
+        ]))
+    }
+
+    async fn update(
+        &self,
+        id: &SeriesId,
+        key: &SeriesKey,
+        metric_id: &MetricId,
+        notify: Option<()>,
+    ) {
+        let current = SegmentDuration::current();
+        let task = if self.cache.contains_key(&current)
+            && self
+                .cache
+                .get(&current)
+                .unwrap()
+                .read()
+                .unwrap()
+                .contains_key(id)
+        {
+            None
+        } else {
+            let result = self
+                .cache
+                .entry(current)
+                .or_default()
+                .write()
+                .unwrap()
+                .insert(*id, key.clone());
+
+            match result {
+                Some(_) => None,
+                None => Some(Task::Series(*id, key.clone(), *metric_id)),
+            }
+        };
+        if let (Some(task), Some(_)) = (task, notify) {
+            self.sender.send(task).await.unwrap();
+        }
+    }
+}
+
+impl TagIndexCache {
+    fn new(storage: TimeMergeStorageRef, sender: Sender<Task>) -> Self {
+        Self {
+            cache: DashMap::new(),
+            write_guard: RwLock::new(HashSet::new()),
+            storage,
+            sender,
+        }
+    }
+
+    async fn load_from_storage(&mut self) {
+        let mut result_stream = self
+            .storage
+            .scan(ScanRequest {
+                range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                predicate: vec![],
+                projections: None,
+            })
+            .await
+            .unwrap();
+        while let Some(item) = result_stream.next().await {
+            let batch = item.unwrap();
+            for index in 0..batch.num_rows() {
+                let metric_id = batch
+                    .column_by_name("metric_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+
+                let tag_name = batch
+                    .column_by_name("tag_name")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .unwrap()
+                    .value(index);
+
+                let tag_value = batch
+                    .column_by_name("tag_value")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<BinaryArray>()
+                    .unwrap()
+                    .value(index);
+
+                let series_id = batch
+                    .column_by_name("series_id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap()
+                    .value(index);
+                self.update(
+                    &SeriesId(series_id),
+                    &vec![tag_name.to_vec()],
+                    &vec![tag_value.to_vec()],
+                    &MetricId(metric_id),
+                    None,
+                )
+                .await;
+            }
+        }
+    }
+
+    fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("metric_id", DataType::UInt64, true),
+            Field::new("tag_name", DataType::Binary, true),
+            Field::new("tag_value", DataType::Binary, true),
+            Field::new("series_id", DataType::UInt64, true),
+        ]))
+    }
+
+    async fn update(
+        &self,
+        series_id: &SeriesId,
+        tag_names: &TagNames,
+        tag_values: &TagValues,
+        metric_id: &MetricId,
+        notify: Option<()>,
+    ) {
+        let current = SegmentDuration::current();
+        let segment_series = SegmentSeries {
+            segment: current,
+            series_id: *series_id,
+        };
+        let task = if self.write_guard.read().unwrap().contains(&segment_series) {
+            None
+        } else {
+            let mut write_guard = self.write_guard.write().unwrap();
+            if write_guard.contains(&segment_series) {
+                None
+            } else {
+                write_guard.insert(segment_series);
+                let cache_lock = self
+                    .cache
+                    .entry(current)
+                    .or_insert_with(|| RwLock::new(HashMap::new()));
+                let mut cache_guard = cache_lock.write().unwrap();
+
+                // __name__ is always at the first
+                let mut names = tag_names.clone();
+                let mut values = tag_values.clone();
+                names.remove(0);
+                values.remove(0);
+                names.iter().zip(values.iter()).for_each(|(name, value)| {
+                    cache_guard
+                        .entry(name.clone())
+                        .or_default()
+                        .entry(value.clone())
+                        .or_default()
+                        .entry(*metric_id)
+                        .or_default()
+                        .insert(*series_id);
+                });
+                Some(Task::TagIndex(*series_id, names, values, *metric_id))
+            }
+        };
+        if let (Some(task), Some(_)) = (task, notify) {
+            self.sender.send(task).await.unwrap();
+        }
+    }
+}
 
-#[derive(Default)]
 pub struct CacheManager {
     metrics: MetricsCache,
     series: SeriesCache,
     tag_index: TagIndexCache,
 }
 
+enum Task {
+    Metric(MetricName),
+    Series(SeriesId, SeriesKey, MetricId),
+    TagIndex(SeriesId, TagNames, TagValues, MetricId),
+}
+
+struct CacheWriter {
+    pub receiver: Receiver<Task>,
+    pub storage: TimeMergeStorageRef,
+    pub schema: Arc<Schema>,
+}
+
+impl CacheWriter {
+    pub fn new(
+        receiver: Receiver<Task>,
+        storage: TimeMergeStorageRef,
+        schema: Arc<Schema>,
+    ) -> Self {
+        Self {
+            receiver,
+            storage,
+            schema,
+        }
+    }
+}
+
+async fn make_storage(
+    runtimes: StorageRuntimes,
+    store: ObjectStoreRef,
+    root_dir: String,
+    num_primary_keys: usize,
+    schema: Arc<Schema>,
+) -> TimeMergeStorageRef {
+    Arc::new(
+        CloudObjectStorage::try_new(
+            root_dir,
+            Duration::from_secs(3600 * 24), // 1 day
+            store.clone(),
+            schema,
+            num_primary_keys, // num_primary_keys
+            StorageConfig::default(),
+            runtimes.clone(),
+        )
+        .await
+        .unwrap(),
+    )
+}
+
 impl CacheManager {
-    pub fn update_metric() -> Option<()> {
-        todo!()
+    pub async fn new(runtimes: StorageRuntimes, store: ObjectStoreRef, root_dir: &str) -> Self {
+        let metrics = {
+            let path = Path::new(root_dir).join("metrics");
+            let root_dir = path.to_string_lossy().to_string();
+            let schema = MetricsCache::schema();
+            let storage =
+                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
+            let (sender, receiver) = mpsc::channel(32);
+            let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
+            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            let mut cache = MetricsCache::new(storage, sender);
+            cache.load_from_storage().await;
+            cache
+        };
+        let series = {
+            let path = Path::new(root_dir).join("series");
+            let root_dir = path.to_string_lossy().to_string();
+            let schema = SeriesCache::schema();
+            let storage =
+                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
+            let (sender, receiver) = mpsc::channel(32);
+            let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
+            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            let mut cache = SeriesCache::new(storage, sender);
+            cache.load_from_storage().await;
+            cache
+        };
+
+        let tag_index = {
+            let path = Path::new(root_dir).join("tag_index");
+            let root_dir = path.to_string_lossy().to_string();
+            let schema = TagIndexCache::schema();
+            let storage = make_storage(runtimes, store, root_dir, 3, schema.clone()).await;
+            let (sender, receiver) = mpsc::channel(32);
+            let writer = CacheWriter::new(receiver, storage.clone(), schema);
+            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            let mut cache = TagIndexCache::new(storage, sender);
+            cache.load_from_storage().await;
+            cache
+        };
+
+        Self {
+            metrics,
+            series,
+            tag_index,
+        }
+    }
+
+    pub async fn update_metric(&self, name: &[u8]) {
+        self.metrics
+            .update(name, &default_field_name(), default_field_type(), Some(()))
+            .await;
+    }
+
+    pub async fn update_series(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
+        self.series.update(id, key, metric_id, Some(())).await;
     }
 
-    pub fn update_series() -> Option<()> {
-        todo!()
+    pub async fn update_tag_index(
+        &self,
+        series_id: &SeriesId,
+        series_key: &SeriesKey,
+        metric_id: &MetricId,
+    ) {
+        self.tag_index
+            .update(
+                series_id,
+                &series_key.names,
+                &series_key.values,
+                metric_id,
+                Some(()),
+            )
+            .await;
     }
 
-    pub fn update_tag_index() -> Option<()> {
-        todo!()
+    async fn execute_write(mut writer: CacheWriter) {
+        while let Some(task) = writer.receiver.recv().await {
+            match task {
+                Task::Metric(name) => {
+                    let arrays: Vec<ArrayRef> = vec![
+                        Arc::new(BinaryArray::from(vec![name.to_byte_slice()])),
+                        Arc::new(UInt64Array::from(vec![hash(&name)])),
+                        Arc::new(BinaryArray::from(
+                            vec![default_field_name().to_byte_slice()],
+                        )),
+                        Arc::new(UInt64Array::from(vec![hash(
+                            default_field_name().to_byte_slice(),
+                        )])),
+                        Arc::new(UInt8Array::from(vec![default_field_type()])),
+                    ];
+                    // Step 4: 构造 RecordBatch
+                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+                    writer
+                        .storage
+                        .write(WriteRequest {
+                            batch,
+                            time_range: (0..10).into(),
+                            enable_check: true,
+                        })
+                        .await
+                        .unwrap();
+                }
+                Task::Series(id, key, metric_id) => {
+                    let offsets: Vec<i32> = vec![0, key.names.len() as i32];
+                    let name_binary_values = key
+                        .names
+                        .iter()
+                        .map(|item| item.as_slice())
+                        .collect::<Vec<_>>();
+                    let name_binary_array = BinaryArray::from_vec(name_binary_values);
+                    let tag_names = ListArray::try_new(
+                        Arc::new(Field::new("item", DataType::Binary, true)),
+                        OffsetBuffer::new(offsets.clone().into()),
+                        Arc::new(name_binary_array),
+                        None,
+                    )
+                    .unwrap();
+
+                    let value_binary_values = key
+                        .names
+                        .iter()
+                        .map(|item| item.as_slice())
+                        .collect::<Vec<_>>();
+                    let value_binary_array = BinaryArray::from_vec(value_binary_values);
+                    let tag_values = ListArray::try_new(
+                        Arc::new(Field::new("item", DataType::Binary, true)),
+                        OffsetBuffer::new(offsets.into()),
+                        Arc::new(value_binary_array),
+                        None,
+                    )
+                    .unwrap();
+
+                    let arrays: Vec<ArrayRef> = vec![
+                        Arc::new(UInt64Array::from(vec![metric_id.0])),
+                        Arc::new(UInt64Array::from(vec![id.0])),
+                        Arc::new(tag_names),
+                        Arc::new(tag_values),
+                    ];
+                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+                    writer
+                        .storage
+                        .write(WriteRequest {
+                            batch,
+                            time_range: (0..10).into(),
+                            enable_check: true,
+                        })
+                        .await
+                        .unwrap();
+                }
+                Task::TagIndex(series_id, names, values, metric_id) => {
+                    let mut metrics_id_builder = UInt64Builder::new();
+                    let mut series_id_builder = UInt64Builder::new();
+                    let mut tag_name_builder = BinaryBuilder::new();
+                    let mut tag_value_builder = BinaryBuilder::new();
+
+                    names.iter().zip(values.iter()).for_each(|(name, value)| {
+                        metrics_id_builder.append_value(metric_id.0);
+                        tag_name_builder.append_value(name.to_byte_slice());
+                        tag_value_builder.append_value(value.to_byte_slice());
+                        series_id_builder.append_value(series_id.0);
+                    });
+                    let arrays: Vec<ArrayRef> = vec![
+                        Arc::new(metrics_id_builder.finish()),
+                        Arc::new(tag_name_builder.finish()),
+                        Arc::new(tag_value_builder.finish()),
+                        Arc::new(series_id_builder.finish()),
+                    ];
+                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+                    writer
+                        .storage
+                        .write(WriteRequest {
+                            batch,
+                            time_range: (0..10).into(),
+                            enable_check: true,
+                        })
+                        .await
+                        .unwrap();
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use horaedb_storage::{
+        storage::ScanRequest,
+        types::{TimeRange, Timestamp},
+    };
+    use object_store::local::LocalFileSystem;
+
+    use super::*;
+    use crate::types::{hash, Label};
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn test_cache_manager_updates() {
+        let rt = Arc::new(Runtime::new().unwrap());
+        let runtimes = StorageRuntimes::new(rt.clone(), rt);
+        let store = Arc::new(LocalFileSystem::new());
+        let root_dir = "/tmp/horaedb".to_string();
+        let cache_manager = CacheManager::new(runtimes, store, root_dir.as_str()).await;
+
+        {
+            // Test update_metric
+            let metric_name = "metric_neo".as_bytes();
+            cache_manager.update_metric(metric_name).await;
+
+            let series_id = SeriesId(11);
+            let lables = vec![
+                Label {
+                    name: b"l1".to_vec(),
+                    value: b"111".to_vec(),
+                },
+                Label {
+                    name: b"l2".to_vec(),
+                    value: b"222".to_vec(),
+                },
+            ];
+            let series_key = SeriesKey::new(Some(metric_name), lables.as_slice());
+
+            // Test update_series
+            cache_manager
+                .update_series(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .await;
+
+            // Test update_tag_index
+            cache_manager
+                .update_tag_index(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .await;
+        }
+
+        {
+            // Test update_metric
+            let metric_name = "metric_neo2".as_bytes();
+            cache_manager.update_metric(metric_name).await;
+
+            let series_id = SeriesId(22);
+            let lables = vec![
+                Label {
+                    name: b"l1".to_vec(),
+                    value: b"111".to_vec(),
+                },
+                Label {
+                    name: b"l2".to_vec(),
+                    value: b"222".to_vec(),
+                },
+            ];
+            let series_key = SeriesKey::new(Some(metric_name), lables.as_slice());
+
+            // Test update_series
+            cache_manager
+                .update_series(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .await;
+
+            // Test update_tag_index
+            cache_manager
+                .update_tag_index(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .await;
+        }
+
+        // sleep 30s
+        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+
+        {
+            println!("metricsssssss.....");
+            let mut result_stream = cache_manager
+                .metrics
+                .storage
+                .scan(ScanRequest {
+                    range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                    predicate: vec![],
+                    projections: None,
+                })
+                .await
+                .unwrap();
+            while let Some(item) = result_stream.next().await {
+                let batch = item.unwrap();
+                println!("=====");
+                println!("batch: {:?}", batch);
+            }
+        }
+        {
+            println!("series.....");
+            let mut result_stream = cache_manager
+                .series
+                .storage
+                .scan(ScanRequest {
+                    range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                    predicate: vec![],
+                    projections: None,
+                })
+                .await
+                .unwrap();
+            while let Some(item) = result_stream.next().await {
+                let batch = item.unwrap();
+                println!("=====");
+                println!("batch: {:?}", batch);
+            }
+        }
+        {
+            println!("tags.....");
+            let mut result_stream = cache_manager
+                .tag_index
+                .storage
+                .scan(ScanRequest {
+                    range: TimeRange::new(Timestamp(0), Timestamp::MAX),
+                    predicate: vec![],
+                    projections: None,
+                })
+                .await
+                .unwrap();
+            while let Some(item) = result_stream.next().await {
+                let batch = item.unwrap();
+                println!("=====");
+                println!("batch: {:?}", batch);
+            }
+        }
     }
 }
+
+// write a test for me
diff --git a/src/metric_engine/src/index/mod.rs b/src/metric_engine/src/index/mod.rs
index 2f1cd35d71..f9987c9e62 100644
--- a/src/metric_engine/src/index/mod.rs
+++ b/src/metric_engine/src/index/mod.rs
@@ -19,7 +19,9 @@ mod cache;
 use std::sync::Arc;
 
 use cache::CacheManager;
-use horaedb_storage::storage::TimeMergeStorageRef;
+use horaedb_storage::storage::{StorageRuntimes, TimeMergeStorageRef};
+use object_store::local::LocalFileSystem;
+use tokio::runtime::Runtime;
 
 use crate::{
     types::{
@@ -34,11 +36,16 @@ pub struct IndexManager {
 }
 
 impl IndexManager {
-    pub fn new(storage: TimeMergeStorageRef) -> Self {
+    pub async fn new() -> Self {
+        // TODO: maybe inialize runtime and store by config?
+        let rt = Arc::new(Runtime::new().unwrap());
+        let runtimes = StorageRuntimes::new(rt.clone(), rt);
+        let store = Arc::new(LocalFileSystem::new());
+        let root_dir = "/tmp/horaedb".to_string();
+
         Self {
             inner: Arc::new(Inner {
-                storage,
-                cache: CacheManager::default(),
+                cache: CacheManager::new(runtimes, store, root_dir.as_str()).await,
             }),
         }
     }
@@ -54,7 +61,7 @@ impl IndexManager {
 
         let series_keys = samples
             .iter()
-            .map(|s| SeriesKey::new(s.name.as_slice(), s.lables.as_slice()))
+            .map(|s| SeriesKey::new(Some(s.name.as_slice()), s.lables.as_slice()))
             .collect::<Vec<_>>();
         let series_ids = series_keys
             .iter()
@@ -86,13 +93,11 @@ impl IndexManager {
                     .create_tag_index(series_id, series_key, metric_id)
             });
 
-        // 5. write all
-        todo!()
+        Ok(())
     }
 }
 
 struct Inner {
-    storage: TimeMergeStorageRef,
     cache: CacheManager,
 }
 
diff --git a/src/metric_engine/src/metric/mod.rs b/src/metric_engine/src/metric/mod.rs
index 69f8e77baf..902b4a9521 100644
--- a/src/metric_engine/src/metric/mod.rs
+++ b/src/metric_engine/src/metric/mod.rs
@@ -26,11 +26,11 @@ pub struct MetricManager {
 }
 
 impl MetricManager {
-    pub fn new(storage: TimeMergeStorageRef) -> Self {
+    pub async fn new(storage: TimeMergeStorageRef) -> Self {
         Self {
             inner: Arc::new(Inner {
                 storage: storage.clone(),
-                index: IndexManager::new(storage),
+                index: IndexManager::new().await,
             }),
         }
     }
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index 9f6f1e35ee..13209b39c1 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -14,25 +14,74 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-use std::{collections::BTreeSet, io::Write, time::Duration};
+use std::{
+    collections::BTreeSet,
+    io::Write,
+    time::{Duration, SystemTime},
+};
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 pub struct MetricId(pub u64);
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub struct SeriesId(pub u64);
-pub type SegmentDuration = Duration;
 pub type MetricName = Vec<u8>;
 pub type FieldName = Vec<u8>;
-pub type FieldType = usize;
+pub type FieldType = u8;
 pub type TagName = Vec<u8>;
 pub type TagValue = Vec<u8>;
+pub type TagNames = Vec<Vec<u8>>;
+pub type TagValues = Vec<Vec<u8>>;
 
+#[inline]
+pub fn default_field_name() -> Vec<u8> {
+    b"value".to_vec()
+}
+
+#[inline]
+pub fn default_field_type() -> u8 {
+    0
+}
+
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
+pub struct SegmentDuration(Duration);
+
+impl SegmentDuration {
+    const ONE_DAY: Duration = Duration::from_secs(24 * 60 * 60);
+
+    pub fn current() -> Self {
+        let now = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        Self(Duration::from_secs(
+            now / SegmentDuration::ONE_DAY.as_secs() * SegmentDuration::ONE_DAY.as_secs(),
+        ))
+    }
+
+    pub fn from(some: Duration) -> Self {
+        Self(Duration::from_secs(
+            some.as_secs() / SegmentDuration::ONE_DAY.as_secs()
+                * SegmentDuration::ONE_DAY.as_secs(),
+        ))
+    }
+}
 #[derive(PartialEq, PartialOrd, Eq, Ord, Clone)]
 pub struct Label {
     pub name: Vec<u8>,
     pub value: Vec<u8>,
 }
 
+impl std::fmt::Debug for Label {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Label {{ name: {:?}, value: {:?} }}",
+            String::from_utf8(self.name.clone()).unwrap(),
+            String::from_utf8(self.value.clone()).unwrap(),
+        )
+    }
+}
+
 /// This is the main struct used for write, optional values will be filled in
 /// different modules.
 pub struct Sample {
@@ -46,32 +95,40 @@ pub struct Sample {
     pub series_id: Option<SeriesId>,
 }
 
-pub struct SeriesKey(Vec<Label>);
+#[derive(Clone, Debug)]
+pub struct SeriesKey {
+    pub names: TagNames,
+    pub values: TagValues,
+}
 
 impl SeriesKey {
-    pub fn new(metric_name: &[u8], lables: &[Label]) -> Self {
+    pub fn new(metric_name: Option<&[u8]>, lables: &[Label]) -> Self {
         let mut set: BTreeSet<Label> = BTreeSet::new();
         lables.iter().for_each(|item| {
             set.insert(item.clone());
         });
-        set.insert(Label {
-            name: String::from("__name__").into(),
-            value: metric_name.to_vec(),
-        });
 
-        Self(set.into_iter().collect::<Vec<Label>>())
+        let mut names = set.iter().map(|item| item.name.clone()).collect::<Vec<_>>();
+        let mut values = set
+            .iter()
+            .map(|item| item.value.clone())
+            .collect::<Vec<_>>();
+        if let Some(metric_name) = metric_name {
+            names.insert(0, String::from("__name__").into());
+            values.insert(0, metric_name.to_vec());
+        }
+        Self { names, values }
     }
 
     pub fn make_bytes(&self) -> Vec<u8> {
         let mut series_bytes: Vec<u8> = Vec::new();
-        self.0.iter().for_each(|item| {
-            series_bytes
-                .write_all(item.name.as_slice())
-                .expect("can write");
-            series_bytes
-                .write_all(item.value.as_slice())
-                .expect("can write");
-        });
+        self.names
+            .iter()
+            .zip(self.values.iter())
+            .for_each(|(name, value)| {
+                series_bytes.write_all(name.as_slice()).expect("can write");
+                series_bytes.write_all(value.as_slice()).expect("can write");
+            });
         series_bytes
     }
 }

From 5bdbf300ec1ef38fc3a880af50df7fde15747e49 Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Fri, 24 Jan 2025 00:00:16 -0600
Subject: [PATCH 3/9] add index cache layer

---
 src/metric_engine/src/index/cache.rs | 761 ++++++++++++++++-----------
 src/metric_engine/src/index/mod.rs   |  75 +--
 src/metric_engine/src/metric/mod.rs  |   7 +-
 src/metric_engine/src/types.rs       |  42 +-
 4 files changed, 519 insertions(+), 366 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index 480d28686b..cc8971b1be 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -1,4 +1,3 @@
-use core::num;
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -24,40 +23,53 @@ use std::{
 
 use arrow::{
     array::{
-        Array, ArrayRef, BinaryArray, BinaryBuilder, Int32BufferBuilder, ListArray, ListBuilder,
-        UInt64Array, UInt64Builder, UInt8Array,
+        Array, ArrayRef, BinaryArray, BinaryBuilder, ListArray, UInt64Array, UInt64Builder,
+        UInt8Array,
     },
     buffer::OffsetBuffer,
-    datatypes::{BinaryType, DataType, Field, Schema, ToByteSlice},
+    datatypes::{DataType, Field, Schema, ToByteSlice},
     record_batch::RecordBatch,
 };
 use dashmap::DashMap;
-use futures::{Stream, StreamExt};
+use futures::StreamExt;
 use horaedb_storage::{
     config::StorageConfig,
     storage::{
-        CloudObjectStorage, ScanRequest, StorageRuntimes, TimeMergeStorage, TimeMergeStorageRef,
-        WriteRequest,
+        CloudObjectStorage, ScanRequest, StorageRuntimes, TimeMergeStorageRef, WriteRequest,
     },
     types::{ObjectStoreRef, TimeRange, Timestamp},
 };
-use tokio::{
-    runtime::Runtime,
-    sync::mpsc::{self, Receiver, Sender},
-};
+use tokio::sync::mpsc::{self, Receiver, Sender};
 
 use crate::types::{
-    default_field_name, default_field_type, hash, FieldName, FieldType, Label, MetricId,
-    MetricName, SegmentDuration, SeriesId, SeriesKey, TagName, TagNames, TagValue, TagValues,
+    hash, FieldName, FieldType, Label, MetricId, MetricName, SegmentDuration, SeriesId, SeriesKey,
+    TagName, TagNames, TagValue, TagValues, DEFAULT_FIELD_NAME, DEFAULT_FIELD_TYPE,
 };
 
+const COLUMN_METRIC_NAME: &str = "metric_name";
+const COLUMN_METRIC_ID: &str = "metric_id";
+const COLUMN_SERIES_ID: &str = "series_id";
+const COLUMN_FIELD_ID: &str = "field_id";
+const COLUMN_FIELD_NAME: &str = "field_name";
+const COLUMN_FIELD_TYPE: &str = "field_type";
+const COLUMN_TAG_NAMES: &str = "tag_names";
+const COLUMN_TAG_VALUES: &str = "tag_values";
+const COLUMN_TAG_NAME: &str = "tag_name";
+const COLUMN_TAG_VALUE: &str = "tag_value";
+const COLUMN_TAG_ITEM: &str = "item";
+
+type ConcurrentMetricMap = RwLock<HashMap<MetricName, (FieldName, FieldType)>>;
+type ConcurrentSeriesMap = RwLock<HashMap<SeriesId, SeriesKey>>;
+type ConcurrentTagKVMap =
+    RwLock<HashMap<TagName, HashMap<TagValue, HashMap<MetricId, HashSet<SeriesId>>>>>;
+
 struct MetricsCache {
-    cache: DashMap<SegmentDuration, RwLock<HashMap<MetricName, (FieldName, FieldType)>>>,
+    cache: DashMap<SegmentDuration, ConcurrentMetricMap>,
     pub storage: TimeMergeStorageRef,
     sender: Sender<Task>,
 }
 struct SeriesCache {
-    cache: DashMap<SegmentDuration, RwLock<HashMap<SeriesId, SeriesKey>>>,
+    cache: DashMap<SegmentDuration, ConcurrentSeriesMap>,
     pub storage: TimeMergeStorageRef,
     sender: Sender<Task>,
 }
@@ -69,12 +81,9 @@ struct SegmentSeries {
 }
 
 struct TagIndexCache {
-    cache: DashMap<
-        SegmentDuration,
-        RwLock<HashMap<TagName, HashMap<TagValue, HashMap<MetricId, HashSet<SeriesId>>>>>,
-    >,
-    write_guard: RwLock<HashSet<SegmentSeries>>,
-    pub storage: TimeMergeStorageRef,
+    cache: DashMap<SegmentDuration, ConcurrentTagKVMap>,
+    series_records: RwLock<HashSet<SegmentSeries>>, // used for
+    storage: TimeMergeStorageRef,
     sender: Sender<Task>,
 }
 
@@ -87,6 +96,52 @@ impl MetricsCache {
         }
     }
 
+    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (&[u8], &[u8], u8, u64, u64) {
+        let metric_name = batch
+            .column_by_name(COLUMN_METRIC_NAME)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .value(index);
+
+        let field_name = batch
+            .column_by_name(COLUMN_FIELD_NAME)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .value(index);
+
+        let field_type = batch
+            .column_by_name(COLUMN_FIELD_TYPE)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt8Array>()
+            .unwrap()
+            .value(index);
+
+        // TODO: why not put in cache?
+        let filed_id = batch
+            .column_by_name(COLUMN_FIELD_ID)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap()
+            .value(index);
+
+        // TODO: why not put in cache?
+        let metric_id = batch
+            .column_by_name(COLUMN_METRIC_ID)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap()
+            .value(index);
+
+        (metric_name, field_name, field_type, filed_id, metric_id)
+    }
+
     async fn load_from_storage(&mut self) {
         let mut result_stream = self
             .storage
@@ -100,66 +155,26 @@ impl MetricsCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.unwrap();
             for index in 0..batch.num_rows() {
-                let metric_name = batch
-                    .column_by_name("metric_name")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .unwrap()
-                    .value(index);
-
-                let field_name = batch
-                    .column_by_name("field_name")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .unwrap()
-                    .value(index);
-
-                let field_type = batch
-                    .column_by_name("field_type")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt8Array>()
-                    .unwrap()
-                    .value(index);
-
-                // TODO: why not put in cache?
-                let _filed_id = batch
-                    .column_by_name("field_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
-
-                // TODO: why not put in cache?
-                let _metric_id = batch
-                    .column_by_name("metric_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
-
-                self.update(metric_name, field_name, field_type, None).await;
+                let (metric_name, field_name, field_type, _, _) =
+                    MetricsCache::parse_record_batch(&batch, index);
+                self.update(metric_name, field_name, field_type);
             }
         }
     }
 
     fn schema() -> Arc<Schema> {
         Arc::new(Schema::new(vec![
-            Field::new("metric_name", DataType::Binary, true),
-            Field::new("metric_id", DataType::UInt64, true),
-            Field::new("field_name", DataType::Binary, true),
-            Field::new("field_id", DataType::UInt64, true),
-            Field::new("field_type", DataType::UInt8, true),
+            Field::new(COLUMN_METRIC_NAME, DataType::Binary, true),
+            Field::new(COLUMN_METRIC_ID, DataType::UInt64, true),
+            Field::new(COLUMN_FIELD_NAME, DataType::Binary, true),
+            Field::new(COLUMN_FIELD_ID, DataType::UInt64, true),
+            Field::new(COLUMN_FIELD_TYPE, DataType::UInt8, true),
         ]))
     }
 
-    async fn update(&self, name: &[u8], field_name: &[u8], field_type: u8, notify: Option<()>) {
-        let current = SegmentDuration::current();
-        let task = if self.cache.contains_key(&current)
+    fn update(&self, name: &[u8], field_name: &[u8], field_type: u8) -> bool {
+        let current = SegmentDuration::current_date();
+        if self.cache.contains_key(&current)
             && self
                 .cache
                 .get(&current)
@@ -168,7 +183,7 @@ impl MetricsCache {
                 .unwrap()
                 .contains_key(name)
         {
-            None
+            false
         } else {
             let result = self
                 .cache
@@ -178,16 +193,16 @@ impl MetricsCache {
                 .unwrap()
                 .insert(name.to_vec(), (field_name.to_vec(), field_type));
 
-            match result {
-                Some(_) => None,
-                None => Some(Task::Metric(name.to_vec())),
-            }
-        };
-
-        if let (Some(task), Some(_)) = (task, notify) {
-            self.sender.send(task).await.unwrap();
+            result.is_none()
         }
     }
+
+    async fn notify_write(&self, name: &[u8], field_name: &[u8], field_type: u8) {
+        self.sender
+            .send(Task::Metric(name.to_vec(), field_name.to_vec(), field_type))
+            .await
+            .unwrap();
+    }
 }
 
 impl SeriesCache {
@@ -199,6 +214,53 @@ impl SeriesCache {
         }
     }
 
+    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (u64, Vec<Vec<u8>>, Vec<Vec<u8>>) {
+        let series_id = batch
+            .column_by_name(COLUMN_SERIES_ID)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap()
+            .value(index);
+
+        let tag_names = {
+            let tag_name_array = batch
+                .column_by_name(COLUMN_TAG_NAMES)
+                .unwrap()
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap()
+                .value(index);
+            let tag_names = tag_name_array
+                .as_any()
+                .downcast_ref::<BinaryArray>()
+                .expect("List elements should be BinaryArray");
+            tag_names
+                .iter()
+                .map(|item| item.unwrap().to_vec())
+                .collect::<Vec<_>>()
+        };
+
+        let tag_values = {
+            let tag_value_array = batch
+                .column_by_name(COLUMN_TAG_VALUES)
+                .unwrap()
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap()
+                .value(index);
+            let tag_values = tag_value_array
+                .as_any()
+                .downcast_ref::<BinaryArray>()
+                .expect("List elements should be BinaryArray");
+            tag_values
+                .iter()
+                .map(|item| item.unwrap().to_vec())
+                .collect::<Vec<_>>()
+        };
+        (series_id, tag_names, tag_values)
+    }
+
     async fn load_from_storage(&mut self) {
         let mut result_stream = self
             .storage
@@ -212,94 +274,47 @@ impl SeriesCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.unwrap();
             for index in 0..batch.num_rows() {
-                let metric_id = batch
-                    .column_by_name("metric_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
-
-                let series_id = batch
-                    .column_by_name("series_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
-
-                let tag_name_array = batch
-                    .column_by_name("tag_names")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<ListArray>()
-                    .unwrap()
-                    .value(index);
-                let tag_names = tag_name_array
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .expect("List elements should be BinaryArray");
-
-                let names = tag_names
-                    .iter()
-                    .map(|item| item.unwrap().to_vec())
-                    .collect::<Vec<_>>();
-
-                let tag_value_array = batch
-                    .column_by_name("tag_values")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<ListArray>()
-                    .unwrap()
-                    .value(index);
-                let tag_values = tag_value_array
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .expect("List elements should be BinaryArray");
-
-                let values = tag_values
-                    .iter()
-                    .map(|item| item.unwrap().to_vec())
-                    .collect::<Vec<_>>();
-
-                let labels = names
+                let (series_id, tag_names, tag_values) =
+                    SeriesCache::parse_record_batch(&batch, index);
+                let labels = tag_names
                     .into_iter()
-                    .zip(values.into_iter())
+                    .zip(tag_values.into_iter())
                     .map(|(name, value)| Label { name, value })
                     .collect::<Vec<_>>();
                 let key = SeriesKey::new(None, labels.as_slice());
-                self.update(&SeriesId(series_id), &key, &MetricId(metric_id), None)
-                    .await;
+                self.update(&SeriesId(series_id), &key);
             }
         }
     }
 
     pub fn schema() -> Arc<Schema> {
         Arc::new(Schema::new(vec![
-            Field::new("metric_id", DataType::UInt64, true),
-            Field::new("series_id", DataType::UInt64, true),
+            Field::new(COLUMN_METRIC_ID, DataType::UInt64, true),
+            Field::new(COLUMN_SERIES_ID, DataType::UInt64, true),
             Field::new(
-                "tag_names",
-                DataType::List(Arc::new(Field::new("item", DataType::Binary, true))),
+                COLUMN_TAG_NAMES,
+                DataType::List(Arc::new(Field::new(
+                    COLUMN_TAG_ITEM,
+                    DataType::Binary,
+                    true,
+                ))),
                 true,
             ),
             Field::new(
-                "tag_values",
-                DataType::List(Arc::new(Field::new("item", DataType::Binary, true))),
+                COLUMN_TAG_VALUES,
+                DataType::List(Arc::new(Field::new(
+                    COLUMN_TAG_ITEM,
+                    DataType::Binary,
+                    true,
+                ))),
                 true,
             ),
         ]))
     }
 
-    async fn update(
-        &self,
-        id: &SeriesId,
-        key: &SeriesKey,
-        metric_id: &MetricId,
-        notify: Option<()>,
-    ) {
-        let current = SegmentDuration::current();
-        let task = if self.cache.contains_key(&current)
+    fn update(&self, id: &SeriesId, key: &SeriesKey) -> bool {
+        let current = SegmentDuration::current_date();
+        if self.cache.contains_key(&current)
             && self
                 .cache
                 .get(&current)
@@ -308,7 +323,7 @@ impl SeriesCache {
                 .unwrap()
                 .contains_key(id)
         {
-            None
+            false
         } else {
             let result = self
                 .cache
@@ -318,27 +333,64 @@ impl SeriesCache {
                 .unwrap()
                 .insert(*id, key.clone());
 
-            match result {
-                Some(_) => None,
-                None => Some(Task::Series(*id, key.clone(), *metric_id)),
-            }
-        };
-        if let (Some(task), Some(_)) = (task, notify) {
-            self.sender.send(task).await.unwrap();
+            result.is_none()
         }
     }
+
+    async fn notify_write(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
+        self.sender
+            .send(Task::Series(*id, key.clone(), *metric_id))
+            .await
+            .unwrap();
+    }
 }
 
 impl TagIndexCache {
     fn new(storage: TimeMergeStorageRef, sender: Sender<Task>) -> Self {
         Self {
             cache: DashMap::new(),
-            write_guard: RwLock::new(HashSet::new()),
+            series_records: RwLock::new(HashSet::new()),
             storage,
             sender,
         }
     }
 
+    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (u64, &[u8], &[u8], u64) {
+        let metric_id = batch
+            .column_by_name(COLUMN_METRIC_ID)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap()
+            .value(index);
+
+        let tag_name = batch
+            .column_by_name(COLUMN_TAG_NAME)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .value(index);
+
+        let tag_value = batch
+            .column_by_name(COLUMN_TAG_VALUE)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .value(index);
+
+        let series_id = batch
+            .column_by_name(COLUMN_SERIES_ID)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap()
+            .value(index);
+
+        (metric_id, tag_name, tag_value, series_id)
+    }
+
     async fn load_from_storage(&mut self) {
         let mut result_stream = self
             .storage
@@ -352,90 +404,56 @@ impl TagIndexCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.unwrap();
             for index in 0..batch.num_rows() {
-                let metric_id = batch
-                    .column_by_name("metric_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
-
-                let tag_name = batch
-                    .column_by_name("tag_name")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .unwrap()
-                    .value(index);
-
-                let tag_value = batch
-                    .column_by_name("tag_value")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<BinaryArray>()
-                    .unwrap()
-                    .value(index);
-
-                let series_id = batch
-                    .column_by_name("series_id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<UInt64Array>()
-                    .unwrap()
-                    .value(index);
+                let (series_id, tag_name, tag_value, metric_id) =
+                    TagIndexCache::parse_record_batch(&batch, index);
                 self.update(
                     &SeriesId(series_id),
                     &vec![tag_name.to_vec()],
                     &vec![tag_value.to_vec()],
                     &MetricId(metric_id),
-                    None,
-                )
-                .await;
+                );
             }
         }
     }
 
     fn schema() -> Arc<Schema> {
         Arc::new(Schema::new(vec![
-            Field::new("metric_id", DataType::UInt64, true),
-            Field::new("tag_name", DataType::Binary, true),
-            Field::new("tag_value", DataType::Binary, true),
-            Field::new("series_id", DataType::UInt64, true),
+            Field::new(COLUMN_METRIC_ID, DataType::UInt64, true),
+            Field::new(COLUMN_TAG_NAME, DataType::Binary, true),
+            Field::new(COLUMN_TAG_VALUE, DataType::Binary, true),
+            Field::new(COLUMN_SERIES_ID, DataType::UInt64, true),
         ]))
     }
 
-    async fn update(
+    fn update(
         &self,
         series_id: &SeriesId,
         tag_names: &TagNames,
         tag_values: &TagValues,
         metric_id: &MetricId,
-        notify: Option<()>,
-    ) {
-        let current = SegmentDuration::current();
+    ) -> bool {
+        let current = SegmentDuration::current_date();
         let segment_series = SegmentSeries {
             segment: current,
             series_id: *series_id,
         };
-        let task = if self.write_guard.read().unwrap().contains(&segment_series) {
-            None
+        if self
+            .series_records
+            .read()
+            .unwrap()
+            .contains(&segment_series)
+        {
+            false
         } else {
-            let mut write_guard = self.write_guard.write().unwrap();
-            if write_guard.contains(&segment_series) {
-                None
+            let mut series_records = self.series_records.write().unwrap();
+            if series_records.contains(&segment_series) {
+                false
             } else {
-                write_guard.insert(segment_series);
-                let cache_lock = self
-                    .cache
-                    .entry(current)
-                    .or_insert_with(|| RwLock::new(HashMap::new()));
+                series_records.insert(segment_series);
+                let cache_lock = self.cache.entry(current).or_default();
                 let mut cache_guard = cache_lock.write().unwrap();
 
-                // __name__ is always at the first
-                let mut names = tag_names.clone();
-                let mut values = tag_values.clone();
-                names.remove(0);
-                values.remove(0);
+                let (names, values) = remove_default_tag(tag_names.clone(), tag_values.clone());
                 names.iter().zip(values.iter()).for_each(|(name, value)| {
                     cache_guard
                         .entry(name.clone())
@@ -446,13 +464,28 @@ impl TagIndexCache {
                         .or_default()
                         .insert(*series_id);
                 });
-                Some(Task::TagIndex(*series_id, names, values, *metric_id))
+                true
             }
-        };
-        if let (Some(task), Some(_)) = (task, notify) {
-            self.sender.send(task).await.unwrap();
         }
     }
+
+    async fn notify_write(
+        &self,
+        series_id: &SeriesId,
+        tag_names: &TagNames,
+        tag_values: &TagValues,
+        metric_id: &MetricId,
+    ) {
+        self.sender
+            .send(Task::TagIndex(
+                *series_id,
+                tag_names.clone(),
+                tag_values.clone(),
+                *metric_id,
+            ))
+            .await
+            .unwrap();
+    }
 }
 
 pub struct CacheManager {
@@ -462,7 +495,7 @@ pub struct CacheManager {
 }
 
 enum Task {
-    Metric(MetricName),
+    Metric(MetricName, FieldName, FieldType),
     Series(SeriesId, SeriesKey, MetricId),
     TagIndex(SeriesId, TagNames, TagValues, MetricId),
 }
@@ -517,7 +550,7 @@ impl CacheManager {
             let schema = MetricsCache::schema();
             let storage =
                 make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
-            let (sender, receiver) = mpsc::channel(32);
+            let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = MetricsCache::new(storage, sender);
@@ -530,7 +563,7 @@ impl CacheManager {
             let schema = SeriesCache::schema();
             let storage =
                 make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
-            let (sender, receiver) = mpsc::channel(32);
+            let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = SeriesCache::new(storage, sender);
@@ -543,7 +576,7 @@ impl CacheManager {
             let root_dir = path.to_string_lossy().to_string();
             let schema = TagIndexCache::schema();
             let storage = make_storage(runtimes, store, root_dir, 3, schema.clone()).await;
-            let (sender, receiver) = mpsc::channel(32);
+            let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema);
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = TagIndexCache::new(storage, sender);
@@ -559,13 +592,21 @@ impl CacheManager {
     }
 
     pub async fn update_metric(&self, name: &[u8]) {
-        self.metrics
-            .update(name, &default_field_name(), default_field_type(), Some(()))
-            .await;
+        let updated = self
+            .metrics
+            .update(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE);
+        if updated {
+            self.metrics
+                .notify_write(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE)
+                .await;
+        }
     }
 
     pub async fn update_series(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
-        self.series.update(id, key, metric_id, Some(())).await;
+        let updated = self.series.update(id, key);
+        if updated {
+            self.series.notify_write(id, key, metric_id).await;
+        }
     }
 
     pub async fn update_tag_index(
@@ -574,33 +615,29 @@ impl CacheManager {
         series_key: &SeriesKey,
         metric_id: &MetricId,
     ) {
-        self.tag_index
-            .update(
-                series_id,
-                &series_key.names,
-                &series_key.values,
-                metric_id,
-                Some(()),
-            )
-            .await;
+        let updated =
+            self.tag_index
+                .update(series_id, &series_key.names, &series_key.values, metric_id);
+        if updated {
+            self.tag_index
+                .notify_write(series_id, &series_key.names, &series_key.values, metric_id)
+                .await;
+        }
     }
 
     async fn execute_write(mut writer: CacheWriter) {
+        // TODO: use try_recv to accumulated a number of tasks and handle them in one
+        // batch
         while let Some(task) = writer.receiver.recv().await {
             match task {
-                Task::Metric(name) => {
+                Task::Metric(name, field_name, field_type) => {
                     let arrays: Vec<ArrayRef> = vec![
                         Arc::new(BinaryArray::from(vec![name.to_byte_slice()])),
                         Arc::new(UInt64Array::from(vec![hash(&name)])),
-                        Arc::new(BinaryArray::from(
-                            vec![default_field_name().to_byte_slice()],
-                        )),
-                        Arc::new(UInt64Array::from(vec![hash(
-                            default_field_name().to_byte_slice(),
-                        )])),
-                        Arc::new(UInt8Array::from(vec![default_field_type()])),
+                        Arc::new(BinaryArray::from(vec![field_name.to_byte_slice()])),
+                        Arc::new(UInt64Array::from(vec![hash(field_name.to_byte_slice())])),
+                        Arc::new(UInt8Array::from(vec![field_type])),
                     ];
-                    // Step 4: 构造 RecordBatch
                     let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
                     writer
                         .storage
@@ -614,39 +651,43 @@ impl CacheManager {
                 }
                 Task::Series(id, key, metric_id) => {
                     let offsets: Vec<i32> = vec![0, key.names.len() as i32];
-                    let name_binary_values = key
-                        .names
-                        .iter()
-                        .map(|item| item.as_slice())
-                        .collect::<Vec<_>>();
-                    let name_binary_array = BinaryArray::from_vec(name_binary_values);
-                    let tag_names = ListArray::try_new(
-                        Arc::new(Field::new("item", DataType::Binary, true)),
-                        OffsetBuffer::new(offsets.clone().into()),
-                        Arc::new(name_binary_array),
-                        None,
-                    )
-                    .unwrap();
-
-                    let value_binary_values = key
-                        .names
-                        .iter()
-                        .map(|item| item.as_slice())
-                        .collect::<Vec<_>>();
-                    let value_binary_array = BinaryArray::from_vec(value_binary_values);
-                    let tag_values = ListArray::try_new(
-                        Arc::new(Field::new("item", DataType::Binary, true)),
-                        OffsetBuffer::new(offsets.into()),
-                        Arc::new(value_binary_array),
-                        None,
-                    )
-                    .unwrap();
+                    let tag_names_array = {
+                        let name_binary_values = key
+                            .names
+                            .iter()
+                            .map(|item| item.as_slice())
+                            .collect::<Vec<_>>();
+                        let name_binary_array = BinaryArray::from_vec(name_binary_values);
+                        ListArray::try_new(
+                            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
+                            OffsetBuffer::new(offsets.clone().into()),
+                            Arc::new(name_binary_array),
+                            None,
+                        )
+                        .unwrap()
+                    };
+
+                    let tag_values_array = {
+                        let value_binary_values = key
+                            .values
+                            .iter()
+                            .map(|item| item.as_slice())
+                            .collect::<Vec<_>>();
+                        let value_binary_array = BinaryArray::from_vec(value_binary_values);
+                        ListArray::try_new(
+                            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
+                            OffsetBuffer::new(offsets.into()),
+                            Arc::new(value_binary_array),
+                            None,
+                        )
+                        .unwrap()
+                    };
 
                     let arrays: Vec<ArrayRef> = vec![
                         Arc::new(UInt64Array::from(vec![metric_id.0])),
                         Arc::new(UInt64Array::from(vec![id.0])),
-                        Arc::new(tag_names),
-                        Arc::new(tag_values),
+                        Arc::new(tag_names_array),
+                        Arc::new(tag_values_array),
                     ];
                     let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
                     writer
@@ -665,6 +706,8 @@ impl CacheManager {
                     let mut tag_name_builder = BinaryBuilder::new();
                     let mut tag_value_builder = BinaryBuilder::new();
 
+                    let (names, values) = remove_default_tag(names, values);
+
                     names.iter().zip(values.iter()).for_each(|(name, value)| {
                         metrics_id_builder.append_value(metric_id.0);
                         tag_name_builder.append_value(name.to_byte_slice());
@@ -693,6 +736,29 @@ impl CacheManager {
     }
 }
 
+fn remove_default_tag(
+    mut names: Vec<Vec<u8>>,
+    mut values: Vec<Vec<u8>>,
+) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
+    let mut to_remove_index = HashSet::new();
+    let mut index = 0;
+    names.retain(|item| {
+        let keep = !item.starts_with(b"__");
+        if !keep {
+            to_remove_index.insert(index);
+        }
+        index += 1;
+        keep
+    });
+    let mut index = 0;
+    values.retain(|_| {
+        let keep = !to_remove_index.contains(&index);
+        index += 1;
+        keep
+    });
+    (names, values)
+}
+
 #[cfg(test)]
 mod tests {
     use horaedb_storage::{
@@ -700,6 +766,7 @@ mod tests {
         types::{TimeRange, Timestamp},
     };
     use object_store::local::LocalFileSystem;
+    use tokio::runtime::Runtime;
 
     use super::*;
     use crate::types::{hash, Label};
@@ -715,66 +782,63 @@ mod tests {
         {
             // Test update_metric
             let metric_name = "metric_neo".as_bytes();
+            let metric_id = MetricId(hash(metric_name));
             cache_manager.update_metric(metric_name).await;
 
             let series_id = SeriesId(11);
             let lables = vec![
                 Label {
-                    name: b"l1".to_vec(),
+                    name: b"label_a".to_vec(),
                     value: b"111".to_vec(),
                 },
                 Label {
-                    name: b"l2".to_vec(),
+                    name: b"label_b".to_vec(),
                     value: b"222".to_vec(),
                 },
             ];
             let series_key = SeriesKey::new(Some(metric_name), lables.as_slice());
-
             // Test update_series
             cache_manager
-                .update_series(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .update_series(&series_id, &series_key, &metric_id)
                 .await;
-
             // Test update_tag_index
             cache_manager
-                .update_tag_index(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .update_tag_index(&series_id, &series_key, &metric_id)
                 .await;
         }
 
         {
             // Test update_metric
             let metric_name = "metric_neo2".as_bytes();
+            let metric_id = MetricId(hash(metric_name));
             cache_manager.update_metric(metric_name).await;
 
             let series_id = SeriesId(22);
             let lables = vec![
                 Label {
-                    name: b"l1".to_vec(),
+                    name: b"label_a".to_vec(),
                     value: b"111".to_vec(),
                 },
                 Label {
-                    name: b"l2".to_vec(),
-                    value: b"222".to_vec(),
+                    name: b"label_c".to_vec(),
+                    value: b"333".to_vec(),
                 },
             ];
             let series_key = SeriesKey::new(Some(metric_name), lables.as_slice());
-
             // Test update_series
             cache_manager
-                .update_series(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .update_series(&series_id, &series_key, &metric_id)
                 .await;
-
             // Test update_tag_index
             cache_manager
-                .update_tag_index(&series_id, &series_key, &MetricId(hash(metric_name)))
+                .update_tag_index(&series_id, &series_key, &metric_id)
                 .await;
         }
 
-        // sleep 30s
-        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+        tokio::time::sleep(std::time::Duration::from_secs(3)).await;
 
+        // scan and test file data
         {
-            println!("metricsssssss.....");
             let mut result_stream = cache_manager
                 .metrics
                 .storage
@@ -785,14 +849,33 @@ mod tests {
                 })
                 .await
                 .unwrap();
-            while let Some(item) = result_stream.next().await {
-                let batch = item.unwrap();
-                println!("=====");
-                println!("batch: {:?}", batch);
+
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                let (metric_name, field_name, field_type, filed_id, metric_id) =
+                    MetricsCache::parse_record_batch(&batch, 0);
+                assert_eq!(metric_name, b"metric_neo");
+                assert_eq!(field_name, b"value");
+                assert_eq!(field_type, 0);
+                assert_eq!(filed_id, 17612580310495814266);
+                assert_eq!(metric_id, 12417319948205937109);
+            }
+
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                let (metric_name, field_name, field_type, filed_id, metric_id) =
+                    MetricsCache::parse_record_batch(&batch, 0);
+                assert_eq!(metric_name, b"metric_neo2");
+                assert_eq!(field_name, b"value");
+                assert_eq!(field_type, 0);
+                assert_eq!(filed_id, 17612580310495814266);
+                assert_eq!(metric_id, 17578343207158939466);
             }
         }
+
         {
-            println!("series.....");
             let mut result_stream = cache_manager
                 .series
                 .storage
@@ -803,14 +886,46 @@ mod tests {
                 })
                 .await
                 .unwrap();
-            while let Some(item) = result_stream.next().await {
-                let batch = item.unwrap();
-                println!("=====");
-                println!("batch: {:?}", batch);
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                let (series_id, tag_names, tag_values) = SeriesCache::parse_record_batch(&batch, 0);
+
+                assert_eq!(series_id, 11);
+                assert_eq!(
+                    tag_names,
+                    vec![
+                        b"__name__".to_vec(),
+                        b"label_a".to_vec(),
+                        b"label_b".to_vec()
+                    ]
+                );
+                assert_eq!(
+                    tag_values,
+                    vec![b"metric_neo".to_vec(), b"111".to_vec(), b"222".to_vec()]
+                );
+            }
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                let (series_id, tag_names, tag_values) = SeriesCache::parse_record_batch(&batch, 0);
+
+                assert_eq!(series_id, 22);
+                assert_eq!(
+                    tag_names,
+                    vec![
+                        b"__name__".to_vec(),
+                        b"label_a".to_vec(),
+                        b"label_c".to_vec()
+                    ]
+                );
+                assert_eq!(
+                    tag_values,
+                    vec![b"metric_neo2".to_vec(), b"111".to_vec(), b"333".to_vec()]
+                );
             }
         }
         {
-            println!("tags.....");
             let mut result_stream = cache_manager
                 .tag_index
                 .storage
@@ -821,10 +936,46 @@ mod tests {
                 })
                 .await
                 .unwrap();
-            while let Some(item) = result_stream.next().await {
-                let batch = item.unwrap();
-                println!("=====");
-                println!("batch: {:?}", batch);
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                assert_eq!(batch.num_rows(), 3);
+                {
+                    let (metric_id, tag_name, tag_value, series_id) =
+                        TagIndexCache::parse_record_batch(&batch, 0);
+                    assert_eq!(metric_id, 12417319948205937109);
+                    assert_eq!(tag_name, b"label_a");
+                    assert_eq!(tag_value, b"111");
+                    assert_eq!(series_id, 11);
+                }
+                {
+                    let (metric_id, tag_name, tag_value, series_id) =
+                        TagIndexCache::parse_record_batch(&batch, 1);
+                    assert_eq!(metric_id, 12417319948205937109);
+                    assert_eq!(tag_name, b"label_b");
+                    assert_eq!(tag_value, b"222");
+                    assert_eq!(series_id, 11);
+                }
+                {
+                    let (metric_id, tag_name, tag_value, series_id) =
+                        TagIndexCache::parse_record_batch(&batch, 2);
+                    assert_eq!(metric_id, 17578343207158939466);
+                    assert_eq!(tag_name, b"label_a");
+                    assert_eq!(tag_value, b"111");
+                    assert_eq!(series_id, 22);
+                }
+            }
+            {
+                let item = result_stream.next().await;
+                let batch = item.unwrap().unwrap();
+                {
+                    let (metric_id, tag_name, tag_value, series_id) =
+                        TagIndexCache::parse_record_batch(&batch, 0);
+                    assert_eq!(metric_id, 17578343207158939466);
+                    assert_eq!(tag_name, b"label_c");
+                    assert_eq!(tag_value, b"333");
+                    assert_eq!(series_id, 22);
+                }
             }
         }
     }
diff --git a/src/metric_engine/src/index/mod.rs b/src/metric_engine/src/index/mod.rs
index f9987c9e62..cdfa237d8e 100644
--- a/src/metric_engine/src/index/mod.rs
+++ b/src/metric_engine/src/index/mod.rs
@@ -19,15 +19,12 @@ mod cache;
 use std::sync::Arc;
 
 use cache::CacheManager;
-use horaedb_storage::storage::{StorageRuntimes, TimeMergeStorageRef};
+use horaedb_storage::storage::StorageRuntimes;
 use object_store::local::LocalFileSystem;
 use tokio::runtime::Runtime;
 
 use crate::{
-    types::{
-        hash, FieldName, FieldType, MetricId, MetricName, Sample, SegmentDuration, SeriesId,
-        SeriesKey, TagName, TagValue,
-    },
+    types::{hash, MetricId, Sample, SeriesId, SeriesKey},
     Result,
 };
 
@@ -37,7 +34,8 @@ pub struct IndexManager {
 
 impl IndexManager {
     pub async fn new() -> Self {
-        // TODO: maybe inialize runtime and store by config?
+        // TODO: maybe initialize runtime and store by config, now just make it
+        // compilable
         let rt = Arc::new(Runtime::new().unwrap());
         let runtimes = StorageRuntimes::new(rt.clone(), rt);
         let store = Arc::new(LocalFileSystem::new());
@@ -53,7 +51,7 @@ impl IndexManager {
     /// Populate series ids from labels.
     /// It will also build inverted index for labels.
     pub async fn populate_series_ids(&self, samples: &mut [Sample]) -> Result<()> {
-        // 1. create metric id and series id
+        // 1.1 create metric id and series id
         let metric_ids = samples
             .iter()
             .map(|s| MetricId(hash(s.name.as_slice())))
@@ -68,30 +66,41 @@ impl IndexManager {
             .map(|e| SeriesId(hash(e.make_bytes().as_slice())))
             .collect::<Vec<_>>();
 
+        // 1.2 populate metric id and series id
         samples.iter_mut().enumerate().for_each(|(i, sample)| {
             sample.name_id = Some(metric_ids[i]);
             sample.series_id = Some(series_ids[i]);
         });
-        // 2. cache metrics
-        samples
-            .iter()
-            .for_each(|s| self.inner.create_metrics(s.name.as_slice()));
-
-        // 3. cache series
-        series_ids
-            .iter()
-            .zip(series_keys.iter())
-            .for_each(|(id, key)| self.inner.create_series(id, key));
 
-        // 4. cache tag index
-        series_ids
-            .iter()
-            .zip(series_keys.iter())
-            .zip(metric_ids.iter())
-            .for_each(|((series_id, series_key), metric_id)| {
-                self.inner
-                    .create_tag_index(series_id, series_key, metric_id)
-            });
+        // 2.1 update cache metrics
+        futures::future::join_all(
+            samples
+                .iter()
+                .map(|s| self.inner.update_metrics(s.name.as_slice())),
+        )
+        .await;
+
+        // 2.2 update cache series
+        futures::future::join_all(
+            series_ids
+                .iter()
+                .zip(series_keys.iter().zip(metric_ids.iter()))
+                .map(|(id, (key, metric_id))| self.inner.update_series(id, key, metric_id)),
+        )
+        .await;
+
+        // 2.3 update cache tag index
+        futures::future::join_all(
+            series_ids
+                .iter()
+                .zip(series_keys.iter())
+                .zip(metric_ids.iter())
+                .map(|((series_id, series_key), metric_id)| {
+                    self.inner
+                        .update_tag_index(series_id, series_key, metric_id)
+                }),
+        )
+        .await;
 
         Ok(())
     }
@@ -102,20 +111,22 @@ struct Inner {
 }
 
 impl Inner {
-    pub fn create_metrics(&self, name: &[u8]) {
-        todo!()
+    pub async fn update_metrics(&self, name: &[u8]) {
+        self.cache.update_metric(name).await
     }
 
-    pub fn create_series(&self, id: &SeriesId, key: &SeriesKey) {
-        todo!()
+    pub async fn update_series(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
+        self.cache.update_series(id, key, metric_id).await;
     }
 
-    pub fn create_tag_index(
+    pub async fn update_tag_index(
         &self,
         series_id: &SeriesId,
         series_key: &SeriesKey,
         metric_id: &MetricId,
     ) {
-        todo!()
+        self.cache
+            .update_tag_index(series_id, series_key, metric_id)
+            .await;
     }
 }
diff --git a/src/metric_engine/src/metric/mod.rs b/src/metric_engine/src/metric/mod.rs
index 902b4a9521..2a8c619fac 100644
--- a/src/metric_engine/src/metric/mod.rs
+++ b/src/metric_engine/src/metric/mod.rs
@@ -37,11 +37,8 @@ impl MetricManager {
 
     /// Populate metric ids from names.
     /// If a name does not exist, it will be created on demand.
-    pub async fn populate_metric_ids(&self, _samples: &mut [Sample]) -> Result<()> {
-        // 1. call index manager to create index
-
-        // 2. call data manager to write samples
-        todo!()
+    pub async fn populate_metric_ids(&self, samples: &mut [Sample]) -> Result<()> {
+        self.inner.populate_metric_ids(samples).await
     }
 }
 
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index 13209b39c1..69551bdb35 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -32,15 +32,9 @@ pub type TagValue = Vec<u8>;
 pub type TagNames = Vec<Vec<u8>>;
 pub type TagValues = Vec<Vec<u8>>;
 
-#[inline]
-pub fn default_field_name() -> Vec<u8> {
-    b"value".to_vec()
-}
-
-#[inline]
-pub fn default_field_type() -> u8 {
-    0
-}
+pub const METRIC_NAME: &str = "__name__";
+pub const DEFAULT_FIELD_NAME: &str = "value";
+pub const DEFAULT_FIELD_TYPE: u8 = 0;
 
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub struct SegmentDuration(Duration);
@@ -48,7 +42,7 @@ pub struct SegmentDuration(Duration);
 impl SegmentDuration {
     const ONE_DAY: Duration = Duration::from_secs(24 * 60 * 60);
 
-    pub fn current() -> Self {
+    pub fn current_date() -> Self {
         let now = SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)
             .unwrap()
@@ -57,13 +51,6 @@ impl SegmentDuration {
             now / SegmentDuration::ONE_DAY.as_secs() * SegmentDuration::ONE_DAY.as_secs(),
         ))
     }
-
-    pub fn from(some: Duration) -> Self {
-        Self(Duration::from_secs(
-            some.as_secs() / SegmentDuration::ONE_DAY.as_secs()
-                * SegmentDuration::ONE_DAY.as_secs(),
-        ))
-    }
 }
 #[derive(PartialEq, PartialOrd, Eq, Ord, Clone)]
 pub struct Label {
@@ -103,18 +90,21 @@ pub struct SeriesKey {
 
 impl SeriesKey {
     pub fn new(metric_name: Option<&[u8]>, lables: &[Label]) -> Self {
-        let mut set: BTreeSet<Label> = BTreeSet::new();
+        let mut sorted_key: BTreeSet<&Label> = BTreeSet::new();
         lables.iter().for_each(|item| {
-            set.insert(item.clone());
+            sorted_key.insert(item);
         });
 
-        let mut names = set.iter().map(|item| item.name.clone()).collect::<Vec<_>>();
-        let mut values = set
+        let mut names = sorted_key
+            .iter()
+            .map(|item| item.name.clone())
+            .collect::<Vec<_>>();
+        let mut values = sorted_key
             .iter()
             .map(|item| item.value.clone())
             .collect::<Vec<_>>();
         if let Some(metric_name) = metric_name {
-            names.insert(0, String::from("__name__").into());
+            names.insert(0, METRIC_NAME.as_bytes().to_vec());
             values.insert(0, metric_name.to_vec());
         }
         Self { names, values }
@@ -126,8 +116,12 @@ impl SeriesKey {
             .iter()
             .zip(self.values.iter())
             .for_each(|(name, value)| {
-                series_bytes.write_all(name.as_slice()).expect("can write");
-                series_bytes.write_all(value.as_slice()).expect("can write");
+                series_bytes
+                    .write_all(name.as_slice())
+                    .expect("could write");
+                series_bytes
+                    .write_all(value.as_slice())
+                    .expect("could write");
             });
         series_bytes
     }

From 7ed30e2f0c994f060271e3906c357907da2979c2 Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Fri, 24 Jan 2025 00:01:57 -0600
Subject: [PATCH 4/9] add index cache layer

---
 src/metric_engine/src/index/cache.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index cc8971b1be..4ca47c6dc0 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -82,7 +82,7 @@ struct SegmentSeries {
 
 struct TagIndexCache {
     cache: DashMap<SegmentDuration, ConcurrentTagKVMap>,
-    series_records: RwLock<HashSet<SegmentSeries>>, // used for
+    series_records: RwLock<HashSet<SegmentSeries>>,
     storage: TimeMergeStorageRef,
     sender: Sender<Task>,
 }
@@ -121,7 +121,6 @@ impl MetricsCache {
             .unwrap()
             .value(index);
 
-        // TODO: why not put in cache?
         let filed_id = batch
             .column_by_name(COLUMN_FIELD_ID)
             .unwrap()
@@ -130,7 +129,6 @@ impl MetricsCache {
             .unwrap()
             .value(index);
 
-        // TODO: why not put in cache?
         let metric_id = batch
             .column_by_name(COLUMN_METRIC_ID)
             .unwrap()
@@ -533,7 +531,7 @@ async fn make_storage(
             Duration::from_secs(3600 * 24), // 1 day
             store.clone(),
             schema,
-            num_primary_keys, // num_primary_keys
+            num_primary_keys,
             StorageConfig::default(),
             runtimes.clone(),
         )
@@ -980,5 +978,3 @@ mod tests {
         }
     }
 }
-
-// write a test for me

From a08287a361f3a12de5082055a756d22d42235cff Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Fri, 24 Jan 2025 18:49:55 -0600
Subject: [PATCH 5/9] add index cache layer

---
 src/metric_engine/src/index/cache.rs | 300 +++++++++++++++------------
 src/metric_engine/src/index/mod.rs   |  23 +-
 src/metric_engine/src/metric/mod.rs  |   8 +-
 src/metric_engine/src/types.rs       |   2 +
 4 files changed, 193 insertions(+), 140 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index 4ca47c6dc0..bd6432a72c 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -17,10 +17,11 @@
 use std::{
     collections::{HashMap, HashSet},
     path::Path,
-    sync::{Arc, RwLock},
+    sync::Arc,
     time::Duration,
 };
 
+use anyhow::Context;
 use arrow::{
     array::{
         Array, ArrayRef, BinaryArray, BinaryBuilder, ListArray, UInt64Array, UInt64Builder,
@@ -39,11 +40,15 @@ use horaedb_storage::{
     },
     types::{ObjectStoreRef, TimeRange, Timestamp},
 };
-use tokio::sync::mpsc::{self, Receiver, Sender};
+use tokio::sync::{
+    mpsc::{self, Receiver, Sender},
+    RwLock,
+};
+use tracing::error;
 
 use crate::types::{
-    hash, FieldName, FieldType, Label, MetricId, MetricName, SegmentDuration, SeriesId, SeriesKey,
-    TagName, TagNames, TagValue, TagValues, DEFAULT_FIELD_NAME, DEFAULT_FIELD_TYPE,
+    hash, FieldName, FieldType, Label, MetricId, MetricName, Result, SegmentDuration, SeriesId,
+    SeriesKey, TagName, TagNames, TagValue, TagValues, DEFAULT_FIELD_NAME, DEFAULT_FIELD_TYPE,
 };
 
 const COLUMN_METRIC_NAME: &str = "metric_name";
@@ -96,51 +101,54 @@ impl MetricsCache {
         }
     }
 
-    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (&[u8], &[u8], u8, u64, u64) {
+    fn parse_record_batch(
+        batch: &RecordBatch,
+        index: usize,
+    ) -> Result<(&[u8], &[u8], u8, u64, u64)> {
         let metric_name = batch
             .column_by_name(COLUMN_METRIC_NAME)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<BinaryArray>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let field_name = batch
             .column_by_name(COLUMN_FIELD_NAME)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<BinaryArray>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let field_type = batch
             .column_by_name(COLUMN_FIELD_TYPE)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt8Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let filed_id = batch
             .column_by_name(COLUMN_FIELD_ID)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt64Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let metric_id = batch
             .column_by_name(COLUMN_METRIC_ID)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt64Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
-        (metric_name, field_name, field_type, filed_id, metric_id)
+        Ok((metric_name, field_name, field_type, filed_id, metric_id))
     }
 
-    async fn load_from_storage(&mut self) {
+    async fn load_from_storage(&mut self) -> Result<()> {
         let mut result_stream = self
             .storage
             .scan(ScanRequest {
@@ -148,16 +156,16 @@ impl MetricsCache {
                 predicate: vec![],
                 projections: None,
             })
-            .await
-            .unwrap();
+            .await?;
         while let Some(item) = result_stream.next().await {
-            let batch = item.unwrap();
+            let batch = item.context("get next batch failed")?;
             for index in 0..batch.num_rows() {
                 let (metric_name, field_name, field_type, _, _) =
-                    MetricsCache::parse_record_batch(&batch, index);
-                self.update(metric_name, field_name, field_type);
+                    MetricsCache::parse_record_batch(&batch, index)?;
+                self.update(metric_name, field_name, field_type).await?;
             }
         }
+        Ok(())
     }
 
     fn schema() -> Arc<Schema> {
@@ -170,36 +178,37 @@ impl MetricsCache {
         ]))
     }
 
-    fn update(&self, name: &[u8], field_name: &[u8], field_type: u8) -> bool {
+    async fn update(&self, name: &[u8], field_name: &[u8], field_type: u8) -> Result<bool> {
         let current = SegmentDuration::current_date();
         if self.cache.contains_key(&current)
             && self
                 .cache
                 .get(&current)
-                .unwrap()
+                .context("get key failed")?
                 .read()
-                .unwrap()
+                .await
                 .contains_key(name)
         {
-            false
+            Ok(false)
         } else {
             let result = self
                 .cache
                 .entry(current)
                 .or_default()
                 .write()
-                .unwrap()
+                .await
                 .insert(name.to_vec(), (field_name.to_vec(), field_type));
 
-            result.is_none()
+            Ok(result.is_none())
         }
     }
 
-    async fn notify_write(&self, name: &[u8], field_name: &[u8], field_type: u8) {
+    async fn notify_write(&self, name: &[u8], field_name: &[u8], field_type: u8) -> Result<()> {
         self.sender
             .send(Task::Metric(name.to_vec(), field_name.to_vec(), field_type))
             .await
-            .unwrap();
+            .context("notify write failed.")?;
+        Ok(())
     }
 }
 
@@ -212,40 +221,43 @@ impl SeriesCache {
         }
     }
 
-    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (u64, Vec<Vec<u8>>, Vec<Vec<u8>>) {
+    async fn parse_record_batch(
+        batch: &RecordBatch,
+        index: usize,
+    ) -> Result<(u64, Vec<Vec<u8>>, Vec<Vec<u8>>)> {
         let series_id = batch
             .column_by_name(COLUMN_SERIES_ID)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt64Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let tag_names = {
             let tag_name_array = batch
                 .column_by_name(COLUMN_TAG_NAMES)
-                .unwrap()
+                .context("get column failed")?
                 .as_any()
                 .downcast_ref::<ListArray>()
-                .unwrap()
+                .context("parse column failed")?
                 .value(index);
             let tag_names = tag_name_array
                 .as_any()
                 .downcast_ref::<BinaryArray>()
-                .expect("List elements should be BinaryArray");
+                .context("parse column failed")?;
             tag_names
                 .iter()
-                .map(|item| item.unwrap().to_vec())
+                .map(|item| item.unwrap_or(b"").to_vec())
                 .collect::<Vec<_>>()
         };
 
         let tag_values = {
             let tag_value_array = batch
                 .column_by_name(COLUMN_TAG_VALUES)
-                .unwrap()
+                .context("get column failed")?
                 .as_any()
                 .downcast_ref::<ListArray>()
-                .unwrap()
+                .context("parse column failed")?
                 .value(index);
             let tag_values = tag_value_array
                 .as_any()
@@ -253,13 +265,13 @@ impl SeriesCache {
                 .expect("List elements should be BinaryArray");
             tag_values
                 .iter()
-                .map(|item| item.unwrap().to_vec())
+                .map(|item| item.unwrap_or(b"").to_vec())
                 .collect::<Vec<_>>()
         };
-        (series_id, tag_names, tag_values)
+        Ok((series_id, tag_names, tag_values))
     }
 
-    async fn load_from_storage(&mut self) {
+    async fn load_from_storage(&mut self) -> Result<()> {
         let mut result_stream = self
             .storage
             .scan(ScanRequest {
@@ -267,22 +279,22 @@ impl SeriesCache {
                 predicate: vec![],
                 projections: None,
             })
-            .await
-            .unwrap();
+            .await?;
         while let Some(item) = result_stream.next().await {
-            let batch = item.unwrap();
+            let batch = item.context("get next batch failed.")?;
             for index in 0..batch.num_rows() {
                 let (series_id, tag_names, tag_values) =
-                    SeriesCache::parse_record_batch(&batch, index);
+                    SeriesCache::parse_record_batch(&batch, index).await?;
                 let labels = tag_names
                     .into_iter()
                     .zip(tag_values.into_iter())
                     .map(|(name, value)| Label { name, value })
                     .collect::<Vec<_>>();
                 let key = SeriesKey::new(None, labels.as_slice());
-                self.update(&SeriesId(series_id), &key);
+                self.update(&SeriesId(series_id), &key).await?;
             }
         }
+        Ok(())
     }
 
     pub fn schema() -> Arc<Schema> {
@@ -310,36 +322,42 @@ impl SeriesCache {
         ]))
     }
 
-    fn update(&self, id: &SeriesId, key: &SeriesKey) -> bool {
+    async fn update(&self, id: &SeriesId, key: &SeriesKey) -> Result<bool> {
         let current = SegmentDuration::current_date();
         if self.cache.contains_key(&current)
             && self
                 .cache
                 .get(&current)
-                .unwrap()
+                .context("get key failed")?
                 .read()
-                .unwrap()
+                .await
                 .contains_key(id)
         {
-            false
+            Ok(false)
         } else {
             let result = self
                 .cache
                 .entry(current)
                 .or_default()
                 .write()
-                .unwrap()
+                .await
                 .insert(*id, key.clone());
 
-            result.is_none()
+            Ok(result.is_none())
         }
     }
 
-    async fn notify_write(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
+    async fn notify_write(
+        &self,
+        id: &SeriesId,
+        key: &SeriesKey,
+        metric_id: &MetricId,
+    ) -> Result<()> {
         self.sender
             .send(Task::Series(*id, key.clone(), *metric_id))
             .await
-            .unwrap();
+            .context("notify write failed.")?;
+        Ok(())
     }
 }
 
@@ -353,43 +371,46 @@ impl TagIndexCache {
         }
     }
 
-    fn parse_record_batch(batch: &RecordBatch, index: usize) -> (u64, &[u8], &[u8], u64) {
+    async fn parse_record_batch(
+        batch: &RecordBatch,
+        index: usize,
+    ) -> Result<(u64, &[u8], &[u8], u64)> {
         let metric_id = batch
             .column_by_name(COLUMN_METRIC_ID)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt64Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let tag_name = batch
             .column_by_name(COLUMN_TAG_NAME)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<BinaryArray>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let tag_value = batch
             .column_by_name(COLUMN_TAG_VALUE)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<BinaryArray>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
         let series_id = batch
             .column_by_name(COLUMN_SERIES_ID)
-            .unwrap()
+            .context("get column failed")?
             .as_any()
             .downcast_ref::<UInt64Array>()
-            .unwrap()
+            .context("parse column failed")?
             .value(index);
 
-        (metric_id, tag_name, tag_value, series_id)
+        Ok((metric_id, tag_name, tag_value, series_id))
     }
 
-    async fn load_from_storage(&mut self) {
+    async fn load_from_storage(&mut self) -> Result<()> {
         let mut result_stream = self
             .storage
             .scan(ScanRequest {
@@ -400,18 +421,20 @@ impl TagIndexCache {
             .await
             .unwrap();
         while let Some(item) = result_stream.next().await {
-            let batch = item.unwrap();
+            let batch = item.context("get next batch failed.")?;
             for index in 0..batch.num_rows() {
                 let (series_id, tag_name, tag_value, metric_id) =
-                    TagIndexCache::parse_record_batch(&batch, index);
+                    TagIndexCache::parse_record_batch(&batch, index).await?;
                 self.update(
                     &SeriesId(series_id),
                     &vec![tag_name.to_vec()],
                     &vec![tag_value.to_vec()],
                     &MetricId(metric_id),
-                );
+                )
+                .await?;
             }
         }
+        Ok(())
     }
 
     fn schema() -> Arc<Schema> {
@@ -423,33 +446,28 @@ impl TagIndexCache {
         ]))
     }
 
-    fn update(
+    async fn update(
         &self,
         series_id: &SeriesId,
         tag_names: &TagNames,
         tag_values: &TagValues,
         metric_id: &MetricId,
-    ) -> bool {
+    ) -> Result<bool> {
         let current = SegmentDuration::current_date();
         let segment_series = SegmentSeries {
             segment: current,
             series_id: *series_id,
         };
-        if self
-            .series_records
-            .read()
-            .unwrap()
-            .contains(&segment_series)
-        {
-            false
+        if self.series_records.read().await.contains(&segment_series) {
+            Ok(false)
         } else {
-            let mut series_records = self.series_records.write().unwrap();
+            let mut series_records = self.series_records.write().await;
             if series_records.contains(&segment_series) {
-                false
+                Ok(false)
             } else {
                 series_records.insert(segment_series);
                 let cache_lock = self.cache.entry(current).or_default();
-                let mut cache_guard = cache_lock.write().unwrap();
+                let mut cache_guard = cache_lock.write().await;
 
                 let (names, values) = remove_default_tag(tag_names.clone(), tag_values.clone());
                 names.iter().zip(values.iter()).for_each(|(name, value)| {
@@ -462,7 +480,7 @@ impl TagIndexCache {
                         .or_default()
                         .insert(*series_id);
                 });
-                true
+                Ok(true)
             }
         }
     }
@@ -473,7 +491,7 @@ impl TagIndexCache {
         tag_names: &TagNames,
         tag_values: &TagValues,
         metric_id: &MetricId,
-    ) {
+    ) -> Result<()> {
         self.sender
             .send(Task::TagIndex(
                 *series_id,
@@ -482,7 +500,8 @@ impl TagIndexCache {
                 *metric_id,
             ))
             .await
-            .unwrap();
+            .context("notify write failed.")?;
+        Ok(())
     }
 }
 
@@ -524,8 +543,8 @@ async fn make_storage(
     root_dir: String,
     num_primary_keys: usize,
     schema: Arc<Schema>,
-) -> TimeMergeStorageRef {
-    Arc::new(
+) -> Result<TimeMergeStorageRef> {
+    Ok(Arc::new(
         CloudObjectStorage::try_new(
             root_dir,
             Duration::from_secs(3600 * 24), // 1 day
@@ -535,24 +554,27 @@ async fn make_storage(
             StorageConfig::default(),
             runtimes.clone(),
         )
-        .await
-        .unwrap(),
-    )
+        .await?,
+    ))
 }
 
 impl CacheManager {
-    pub async fn new(runtimes: StorageRuntimes, store: ObjectStoreRef, root_dir: &str) -> Self {
+    pub async fn try_new(
+        runtimes: StorageRuntimes,
+        store: ObjectStoreRef,
+        root_dir: &str,
+    ) -> Result<Self> {
         let metrics = {
             let path = Path::new(root_dir).join("metrics");
             let root_dir = path.to_string_lossy().to_string();
             let schema = MetricsCache::schema();
             let storage =
-                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
+                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = MetricsCache::new(storage, sender);
-            cache.load_from_storage().await;
+            cache.load_from_storage().await?;
             cache
         };
         let series = {
@@ -560,12 +582,12 @@ impl CacheManager {
             let root_dir = path.to_string_lossy().to_string();
             let schema = SeriesCache::schema();
             let storage =
-                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await;
+                make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = SeriesCache::new(storage, sender);
-            cache.load_from_storage().await;
+            cache.load_from_storage().await?;
             cache
         };
 
@@ -573,38 +595,46 @@ impl CacheManager {
             let path = Path::new(root_dir).join("tag_index");
             let root_dir = path.to_string_lossy().to_string();
             let schema = TagIndexCache::schema();
-            let storage = make_storage(runtimes, store, root_dir, 3, schema.clone()).await;
+            let storage = make_storage(runtimes, store, root_dir, 3, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema);
             tokio::spawn(async move { CacheManager::execute_write(writer).await });
             let mut cache = TagIndexCache::new(storage, sender);
-            cache.load_from_storage().await;
+            cache.load_from_storage().await?;
             cache
         };
 
-        Self {
+        Ok(Self {
             metrics,
             series,
             tag_index,
-        }
+        })
     }
 
-    pub async fn update_metric(&self, name: &[u8]) {
+    pub async fn update_metric(&self, name: &[u8]) -> Result<()> {
         let updated = self
             .metrics
-            .update(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE);
+            .update(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE)
+            .await?;
         if updated {
             self.metrics
                 .notify_write(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE)
-                .await;
+                .await?;
         }
+        Ok(())
     }
 
-    pub async fn update_series(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
-        let updated = self.series.update(id, key);
+    pub async fn update_series(
+        &self,
+        id: &SeriesId,
+        key: &SeriesKey,
+        metric_id: &MetricId,
+    ) -> Result<()> {
+        let updated = self.series.update(id, key).await?;
         if updated {
-            self.series.notify_write(id, key, metric_id).await;
+            self.series.notify_write(id, key, metric_id).await?;
         }
+        Ok(())
     }
 
     pub async fn update_tag_index(
@@ -612,15 +642,17 @@ impl CacheManager {
         series_id: &SeriesId,
         series_key: &SeriesKey,
         metric_id: &MetricId,
-    ) {
-        let updated =
-            self.tag_index
-                .update(series_id, &series_key.names, &series_key.values, metric_id);
+    ) -> Result<()> {
+        let updated = self
+            .tag_index
+            .update(series_id, &series_key.names, &series_key.values, metric_id)
+            .await?;
         if updated {
             self.tag_index
                 .notify_write(series_id, &series_key.names, &series_key.values, metric_id)
-                .await;
+                .await?;
         }
+        Ok(())
     }
 
     async fn execute_write(mut writer: CacheWriter) {
@@ -645,7 +677,9 @@ impl CacheManager {
                             enable_check: true,
                         })
                         .await
-                        .unwrap();
+                        .unwrap_or_else(|e| {
+                            error!("write metrics failed: {:?}", e);
+                        });
                 }
                 Task::Series(id, key, metric_id) => {
                     let offsets: Vec<i32> = vec![0, key.names.len() as i32];
@@ -696,7 +730,9 @@ impl CacheManager {
                             enable_check: true,
                         })
                         .await
-                        .unwrap();
+                        .unwrap_or_else(|e| {
+                            error!("write metrics failed: {:?}", e);
+                        });
                 }
                 Task::TagIndex(series_id, names, values, metric_id) => {
                     let mut metrics_id_builder = UInt64Builder::new();
@@ -727,7 +763,9 @@ impl CacheManager {
                             enable_check: true,
                         })
                         .await
-                        .unwrap();
+                        .unwrap_or_else(|e| {
+                            error!("write metrics failed: {:?}", e);
+                        });
                 }
             }
         }
@@ -775,13 +813,15 @@ mod tests {
         let runtimes = StorageRuntimes::new(rt.clone(), rt);
         let store = Arc::new(LocalFileSystem::new());
         let root_dir = "/tmp/horaedb".to_string();
-        let cache_manager = CacheManager::new(runtimes, store, root_dir.as_str()).await;
+        let cache_manager = CacheManager::try_new(runtimes, store, root_dir.as_str())
+            .await
+            .unwrap();
 
         {
             // Test update_metric
             let metric_name = "metric_neo".as_bytes();
             let metric_id = MetricId(hash(metric_name));
-            cache_manager.update_metric(metric_name).await;
+            cache_manager.update_metric(metric_name).await.unwrap();
 
             let series_id = SeriesId(11);
             let lables = vec![
@@ -798,18 +838,20 @@ mod tests {
             // Test update_series
             cache_manager
                 .update_series(&series_id, &series_key, &metric_id)
-                .await;
+                .await
+                .unwrap();
             // Test update_tag_index
             cache_manager
                 .update_tag_index(&series_id, &series_key, &metric_id)
-                .await;
+                .await
+                .unwrap();
         }
 
         {
             // Test update_metric
             let metric_name = "metric_neo2".as_bytes();
             let metric_id = MetricId(hash(metric_name));
-            cache_manager.update_metric(metric_name).await;
+            cache_manager.update_metric(metric_name).await.unwrap();
 
             let series_id = SeriesId(22);
             let lables = vec![
@@ -826,11 +868,13 @@ mod tests {
             // Test update_series
             cache_manager
                 .update_series(&series_id, &series_key, &metric_id)
-                .await;
+                .await
+                .unwrap();
             // Test update_tag_index
             cache_manager
                 .update_tag_index(&series_id, &series_key, &metric_id)
-                .await;
+                .await
+                .unwrap();
         }
 
         tokio::time::sleep(std::time::Duration::from_secs(3)).await;
@@ -852,7 +896,7 @@ mod tests {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
                 let (metric_name, field_name, field_type, filed_id, metric_id) =
-                    MetricsCache::parse_record_batch(&batch, 0);
+                    MetricsCache::parse_record_batch(&batch, 0).unwrap();
                 assert_eq!(metric_name, b"metric_neo");
                 assert_eq!(field_name, b"value");
                 assert_eq!(field_type, 0);
@@ -864,7 +908,7 @@ mod tests {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
                 let (metric_name, field_name, field_type, filed_id, metric_id) =
-                    MetricsCache::parse_record_batch(&batch, 0);
+                    MetricsCache::parse_record_batch(&batch, 0).unwrap();
                 assert_eq!(metric_name, b"metric_neo2");
                 assert_eq!(field_name, b"value");
                 assert_eq!(field_type, 0);
@@ -887,7 +931,8 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (series_id, tag_names, tag_values) = SeriesCache::parse_record_batch(&batch, 0);
+                let (series_id, tag_names, tag_values) =
+                    SeriesCache::parse_record_batch(&batch, 0).await.unwrap();
 
                 assert_eq!(series_id, 11);
                 assert_eq!(
@@ -906,7 +951,8 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (series_id, tag_names, tag_values) = SeriesCache::parse_record_batch(&batch, 0);
+                let (series_id, tag_names, tag_values) =
+                    SeriesCache::parse_record_batch(&batch, 0).await.unwrap();
 
                 assert_eq!(series_id, 22);
                 assert_eq!(
@@ -940,7 +986,7 @@ mod tests {
                 assert_eq!(batch.num_rows(), 3);
                 {
                     let (metric_id, tag_name, tag_value, series_id) =
-                        TagIndexCache::parse_record_batch(&batch, 0);
+                        TagIndexCache::parse_record_batch(&batch, 0).await.unwrap();
                     assert_eq!(metric_id, 12417319948205937109);
                     assert_eq!(tag_name, b"label_a");
                     assert_eq!(tag_value, b"111");
@@ -948,7 +994,7 @@ mod tests {
                 }
                 {
                     let (metric_id, tag_name, tag_value, series_id) =
-                        TagIndexCache::parse_record_batch(&batch, 1);
+                        TagIndexCache::parse_record_batch(&batch, 1).await.unwrap();
                     assert_eq!(metric_id, 12417319948205937109);
                     assert_eq!(tag_name, b"label_b");
                     assert_eq!(tag_value, b"222");
@@ -956,7 +1002,7 @@ mod tests {
                 }
                 {
                     let (metric_id, tag_name, tag_value, series_id) =
-                        TagIndexCache::parse_record_batch(&batch, 2);
+                        TagIndexCache::parse_record_batch(&batch, 2).await.unwrap();
                     assert_eq!(metric_id, 17578343207158939466);
                     assert_eq!(tag_name, b"label_a");
                     assert_eq!(tag_value, b"111");
@@ -968,7 +1014,7 @@ mod tests {
                 let batch = item.unwrap().unwrap();
                 {
                     let (metric_id, tag_name, tag_value, series_id) =
-                        TagIndexCache::parse_record_batch(&batch, 0);
+                        TagIndexCache::parse_record_batch(&batch, 0).await.unwrap();
                     assert_eq!(metric_id, 17578343207158939466);
                     assert_eq!(tag_name, b"label_c");
                     assert_eq!(tag_value, b"333");
diff --git a/src/metric_engine/src/index/mod.rs b/src/metric_engine/src/index/mod.rs
index cdfa237d8e..07eba82866 100644
--- a/src/metric_engine/src/index/mod.rs
+++ b/src/metric_engine/src/index/mod.rs
@@ -33,7 +33,7 @@ pub struct IndexManager {
 }
 
 impl IndexManager {
-    pub async fn new() -> Self {
+    pub async fn try_new() -> Result<Self> {
         // TODO: maybe initialize runtime and store by config, now just make it
         // compilable
         let rt = Arc::new(Runtime::new().unwrap());
@@ -41,11 +41,11 @@ impl IndexManager {
         let store = Arc::new(LocalFileSystem::new());
         let root_dir = "/tmp/horaedb".to_string();
 
-        Self {
+        Ok(Self {
             inner: Arc::new(Inner {
-                cache: CacheManager::new(runtimes, store, root_dir.as_str()).await,
+                cache: CacheManager::try_new(runtimes, store, root_dir.as_str()).await?,
             }),
-        }
+        })
     }
 
     /// Populate series ids from labels.
@@ -111,12 +111,17 @@ struct Inner {
 }
 
 impl Inner {
-    pub async fn update_metrics(&self, name: &[u8]) {
+    pub async fn update_metrics(&self, name: &[u8]) -> Result<()> {
         self.cache.update_metric(name).await
     }
 
-    pub async fn update_series(&self, id: &SeriesId, key: &SeriesKey, metric_id: &MetricId) {
-        self.cache.update_series(id, key, metric_id).await;
+    pub async fn update_series(
+        &self,
+        id: &SeriesId,
+        key: &SeriesKey,
+        metric_id: &MetricId,
+    ) -> Result<()> {
+        self.cache.update_series(id, key, metric_id).await
     }
 
     pub async fn update_tag_index(
@@ -124,9 +129,9 @@ impl Inner {
         series_id: &SeriesId,
         series_key: &SeriesKey,
         metric_id: &MetricId,
-    ) {
+    ) -> Result<()> {
         self.cache
             .update_tag_index(series_id, series_key, metric_id)
-            .await;
+            .await
     }
 }
diff --git a/src/metric_engine/src/metric/mod.rs b/src/metric_engine/src/metric/mod.rs
index 2a8c619fac..bc496e8400 100644
--- a/src/metric_engine/src/metric/mod.rs
+++ b/src/metric_engine/src/metric/mod.rs
@@ -26,13 +26,13 @@ pub struct MetricManager {
 }
 
 impl MetricManager {
-    pub async fn new(storage: TimeMergeStorageRef) -> Self {
-        Self {
+    pub async fn new(storage: TimeMergeStorageRef) -> Result<Self> {
+        Ok(Self {
             inner: Arc::new(Inner {
                 storage: storage.clone(),
-                index: IndexManager::new().await,
+                index: IndexManager::try_new().await?,
             }),
-        }
+        })
     }
 
     /// Populate metric ids from names.
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index 69551bdb35..c5a6910eae 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -20,6 +20,8 @@ use std::{
     time::{Duration, SystemTime},
 };
 
+pub type Result<T> = common::Result<T>;
+
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 pub struct MetricId(pub u64);
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]

From 07e454ce7bacae66560d52a02312a2ccaf397ba1 Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Sat, 25 Jan 2025 01:54:56 -0600
Subject: [PATCH 6/9] add index cache layer

---
 src/metric_engine/src/index/cache.rs | 582 +++++++++++++++++++--------
 src/metric_engine/src/metric/mod.rs  |   2 +-
 src/metric_engine/src/types.rs       |  22 +-
 3 files changed, 416 insertions(+), 190 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index bd6432a72c..3d011b3817 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -18,14 +18,14 @@ use std::{
     collections::{HashMap, HashSet},
     path::Path,
     sync::Arc,
-    time::Duration,
+    time::{Duration, SystemTime},
 };
 
 use anyhow::Context;
 use arrow::{
     array::{
         Array, ArrayRef, BinaryArray, BinaryBuilder, ListArray, UInt64Array, UInt64Builder,
-        UInt8Array,
+        UInt8Array, UInt8Builder,
     },
     buffer::OffsetBuffer,
     datatypes::{DataType, Field, Schema, ToByteSlice},
@@ -40,17 +40,21 @@ use horaedb_storage::{
     },
     types::{ObjectStoreRef, TimeRange, Timestamp},
 };
-use tokio::sync::{
-    mpsc::{self, Receiver, Sender},
-    RwLock,
+use tokio::{
+    sync::{
+        mpsc::{self, Receiver, Sender},
+        RwLock,
+    },
+    time::timeout,
 };
-use tracing::error;
+use tracing::{error, warn};
 
 use crate::types::{
     hash, FieldName, FieldType, Label, MetricId, MetricName, Result, SegmentDuration, SeriesId,
     SeriesKey, TagName, TagNames, TagValue, TagValues, DEFAULT_FIELD_NAME, DEFAULT_FIELD_TYPE,
 };
 
+const COLUMN_DURATION: &str = "duration";
 const COLUMN_METRIC_NAME: &str = "metric_name";
 const COLUMN_METRIC_ID: &str = "metric_id";
 const COLUMN_SERIES_ID: &str = "series_id";
@@ -104,7 +108,7 @@ impl MetricsCache {
     fn parse_record_batch(
         batch: &RecordBatch,
         index: usize,
-    ) -> Result<(&[u8], &[u8], u8, u64, u64)> {
+    ) -> Result<(&[u8], &[u8], u8, u64, u64, u64)> {
         let metric_name = batch
             .column_by_name(COLUMN_METRIC_NAME)
             .context("get column failed")?
@@ -145,7 +149,22 @@ impl MetricsCache {
             .context("parse column failed")?
             .value(index);
 
-        Ok((metric_name, field_name, field_type, filed_id, metric_id))
+        let duration = batch
+            .column_by_name(COLUMN_DURATION)
+            .context("get column failed")?
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .context("parse column failed")?
+            .value(index);
+
+        Ok((
+            metric_name,
+            field_name,
+            field_type,
+            filed_id,
+            metric_id,
+            duration,
+        ))
     }
 
     async fn load_from_storage(&mut self) -> Result<()> {
@@ -160,9 +179,15 @@ impl MetricsCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.context("get next batch failed")?;
             for index in 0..batch.num_rows() {
-                let (metric_name, field_name, field_type, _, _) =
+                let (metric_name, field_name, field_type, _, _, duration) =
                     MetricsCache::parse_record_batch(&batch, index)?;
-                self.update(metric_name, field_name, field_type).await?;
+                self.update(
+                    SegmentDuration::date(Duration::from_millis(duration)),
+                    metric_name,
+                    field_name,
+                    field_type,
+                )
+                .await?;
             }
         }
         Ok(())
@@ -175,15 +200,21 @@ impl MetricsCache {
             Field::new(COLUMN_FIELD_NAME, DataType::Binary, true),
             Field::new(COLUMN_FIELD_ID, DataType::UInt64, true),
             Field::new(COLUMN_FIELD_TYPE, DataType::UInt8, true),
+            Field::new(COLUMN_DURATION, DataType::UInt64, true),
         ]))
     }
 
-    async fn update(&self, name: &[u8], field_name: &[u8], field_type: u8) -> Result<bool> {
-        let current = SegmentDuration::current_date();
-        if self.cache.contains_key(&current)
+    async fn update(
+        &self,
+        date: SegmentDuration,
+        name: &[u8],
+        field_name: &[u8],
+        field_type: u8,
+    ) -> Result<bool> {
+        if self.cache.contains_key(&date)
             && self
                 .cache
-                .get(&current)
+                .get(&date)
                 .context("get key failed")?
                 .read()
                 .await
@@ -193,7 +224,7 @@ impl MetricsCache {
         } else {
             let result = self
                 .cache
-                .entry(current)
+                .entry(date)
                 .or_default()
                 .write()
                 .await
@@ -203,9 +234,20 @@ impl MetricsCache {
         }
     }
 
-    async fn notify_write(&self, name: &[u8], field_name: &[u8], field_type: u8) -> Result<()> {
+    async fn notify_write(
+        &self,
+        current: Duration,
+        name: &[u8],
+        field_name: &[u8],
+        field_type: u8,
+    ) -> Result<()> {
         self.sender
-            .send(Task::Metric(name.to_vec(), field_name.to_vec(), field_type))
+            .send(Task::Metric(
+                current,
+                name.to_vec(),
+                field_name.to_vec(),
+                field_type,
+            ))
             .await
             .context("notify write failed.")?;
         Ok(())
@@ -224,7 +266,7 @@ impl SeriesCache {
     async fn parse_record_batch(
         batch: &RecordBatch,
         index: usize,
-    ) -> Result<(u64, Vec<Vec<u8>>, Vec<Vec<u8>>)> {
+    ) -> Result<(u64, Vec<Vec<u8>>, Vec<Vec<u8>>, u64)> {
         let series_id = batch
             .column_by_name(COLUMN_SERIES_ID)
             .context("get column failed")?
@@ -268,7 +310,15 @@ impl SeriesCache {
                 .map(|item| item.unwrap_or(b"").to_vec())
                 .collect::<Vec<_>>()
         };
-        Ok((series_id, tag_names, tag_values))
+
+        let duration = batch
+            .column_by_name(COLUMN_DURATION)
+            .context("get column failed")?
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .context("parse column failed")?
+            .value(index);
+        Ok((series_id, tag_names, tag_values, duration))
     }
 
     async fn load_from_storage(&mut self) -> Result<()> {
@@ -283,7 +333,7 @@ impl SeriesCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.context("get next batch failed.")?;
             for index in 0..batch.num_rows() {
-                let (series_id, tag_names, tag_values) =
+                let (series_id, tag_names, tag_values, duration) =
                     SeriesCache::parse_record_batch(&batch, index).await?;
                 let labels = tag_names
                     .into_iter()
@@ -291,7 +341,12 @@ impl SeriesCache {
                     .map(|(name, value)| Label { name, value })
                     .collect::<Vec<_>>();
                 let key = SeriesKey::new(None, labels.as_slice());
-                self.update(&SeriesId(series_id), &key).await?;
+                self.update(
+                    SegmentDuration::date(Duration::from_millis(duration)),
+                    &SeriesId(series_id),
+                    &key,
+                )
+                .await?;
             }
         }
         Ok(())
@@ -319,15 +374,15 @@ impl SeriesCache {
                 ))),
                 true,
             ),
+            Field::new(COLUMN_DURATION, DataType::UInt64, true),
         ]))
     }
 
-    async fn update(&self, id: &SeriesId, key: &SeriesKey) -> Result<bool> {
-        let current = SegmentDuration::current_date();
-        if self.cache.contains_key(&current)
+    async fn update(&self, date: SegmentDuration, id: &SeriesId, key: &SeriesKey) -> Result<bool> {
+        if self.cache.contains_key(&date)
             && self
                 .cache
-                .get(&current)
+                .get(&date)
                 .context("get key failed")?
                 .read()
                 .await
@@ -337,7 +392,7 @@ impl SeriesCache {
         } else {
             let result = self
                 .cache
-                .entry(current)
+                .entry(date)
                 .or_default()
                 .write()
                 .await
@@ -349,12 +404,13 @@ impl SeriesCache {
 
     async fn notify_write(
         &self,
+        current: Duration,
         id: &SeriesId,
         key: &SeriesKey,
         metric_id: &MetricId,
     ) -> Result<()> {
         self.sender
-            .send(Task::Series(*id, key.clone(), *metric_id))
+            .send(Task::Series(current, *id, key.clone(), *metric_id))
             .await
             .context("notify write failed.")?;
         Ok(())
@@ -374,7 +430,7 @@ impl TagIndexCache {
     async fn parse_record_batch(
         batch: &RecordBatch,
         index: usize,
-    ) -> Result<(u64, &[u8], &[u8], u64)> {
+    ) -> Result<(u64, &[u8], &[u8], u64, u64)> {
         let metric_id = batch
             .column_by_name(COLUMN_METRIC_ID)
             .context("get column failed")?
@@ -407,7 +463,15 @@ impl TagIndexCache {
             .context("parse column failed")?
             .value(index);
 
-        Ok((metric_id, tag_name, tag_value, series_id))
+        let duration = batch
+            .column_by_name(COLUMN_DURATION)
+            .context("get column failed")?
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .context("parse column failed")?
+            .value(index);
+
+        Ok((metric_id, tag_name, tag_value, series_id, duration))
     }
 
     async fn load_from_storage(&mut self) -> Result<()> {
@@ -423,9 +487,10 @@ impl TagIndexCache {
         while let Some(item) = result_stream.next().await {
             let batch = item.context("get next batch failed.")?;
             for index in 0..batch.num_rows() {
-                let (series_id, tag_name, tag_value, metric_id) =
+                let (series_id, tag_name, tag_value, metric_id, duration) =
                     TagIndexCache::parse_record_batch(&batch, index).await?;
                 self.update(
+                    SegmentDuration::date(Duration::from_millis(duration)),
                     &SeriesId(series_id),
                     &vec![tag_name.to_vec()],
                     &vec![tag_value.to_vec()],
@@ -443,19 +508,20 @@ impl TagIndexCache {
             Field::new(COLUMN_TAG_NAME, DataType::Binary, true),
             Field::new(COLUMN_TAG_VALUE, DataType::Binary, true),
             Field::new(COLUMN_SERIES_ID, DataType::UInt64, true),
+            Field::new(COLUMN_DURATION, DataType::UInt64, true),
         ]))
     }
 
     async fn update(
         &self,
+        date: SegmentDuration,
         series_id: &SeriesId,
         tag_names: &TagNames,
         tag_values: &TagValues,
         metric_id: &MetricId,
     ) -> Result<bool> {
-        let current = SegmentDuration::current_date();
         let segment_series = SegmentSeries {
-            segment: current,
+            segment: date,
             series_id: *series_id,
         };
         if self.series_records.read().await.contains(&segment_series) {
@@ -466,20 +532,25 @@ impl TagIndexCache {
                 Ok(false)
             } else {
                 series_records.insert(segment_series);
-                let cache_lock = self.cache.entry(current).or_default();
+                let cache_lock = self.cache.entry(date).or_default();
                 let mut cache_guard = cache_lock.write().await;
 
-                let (names, values) = remove_default_tag(tag_names.clone(), tag_values.clone());
-                names.iter().zip(values.iter()).for_each(|(name, value)| {
-                    cache_guard
-                        .entry(name.clone())
-                        .or_default()
-                        .entry(value.clone())
-                        .or_default()
-                        .entry(*metric_id)
-                        .or_default()
-                        .insert(*series_id);
-                });
+                let mut tag_names = tag_names.clone();
+                let mut tag_values = tag_values.clone();
+                remove_default_tag(&mut tag_names, &mut tag_values);
+                tag_names
+                    .iter()
+                    .zip(tag_values.iter())
+                    .for_each(|(name, value)| {
+                        cache_guard
+                            .entry(name.clone())
+                            .or_default()
+                            .entry(value.clone())
+                            .or_default()
+                            .entry(*metric_id)
+                            .or_default()
+                            .insert(*series_id);
+                    });
                 Ok(true)
             }
         }
@@ -487,6 +558,7 @@ impl TagIndexCache {
 
     async fn notify_write(
         &self,
+        current: Duration,
         series_id: &SeriesId,
         tag_names: &TagNames,
         tag_values: &TagValues,
@@ -494,6 +566,7 @@ impl TagIndexCache {
     ) -> Result<()> {
         self.sender
             .send(Task::TagIndex(
+                current,
                 *series_id,
                 tag_names.clone(),
                 tag_values.clone(),
@@ -512,9 +585,9 @@ pub struct CacheManager {
 }
 
 enum Task {
-    Metric(MetricName, FieldName, FieldType),
-    Series(SeriesId, SeriesKey, MetricId),
-    TagIndex(SeriesId, TagNames, TagValues, MetricId),
+    Metric(Duration, MetricName, FieldName, FieldType),
+    Series(Duration, SeriesId, SeriesKey, MetricId),
+    TagIndex(Duration, SeriesId, TagNames, TagValues, MetricId),
 }
 
 struct CacheWriter {
@@ -572,7 +645,7 @@ impl CacheManager {
                 make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
-            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            tokio::spawn(async move { execute_write(writer).await });
             let mut cache = MetricsCache::new(storage, sender);
             cache.load_from_storage().await?;
             cache
@@ -585,7 +658,7 @@ impl CacheManager {
                 make_storage(runtimes.clone(), store.clone(), root_dir, 2, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema.clone());
-            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            tokio::spawn(async move { execute_write(writer).await });
             let mut cache = SeriesCache::new(storage, sender);
             cache.load_from_storage().await?;
             cache
@@ -598,7 +671,7 @@ impl CacheManager {
             let storage = make_storage(runtimes, store, root_dir, 3, schema.clone()).await?;
             let (sender, receiver) = mpsc::channel(1024);
             let writer = CacheWriter::new(receiver, storage.clone(), schema);
-            tokio::spawn(async move { CacheManager::execute_write(writer).await });
+            tokio::spawn(async move { execute_write(writer).await });
             let mut cache = TagIndexCache::new(storage, sender);
             cache.load_from_storage().await?;
             cache
@@ -612,13 +685,27 @@ impl CacheManager {
     }
 
     pub async fn update_metric(&self, name: &[u8]) -> Result<()> {
+        let current = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap();
+        let date = SegmentDuration::date(current);
         let updated = self
             .metrics
-            .update(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE)
+            .update(
+                date,
+                name,
+                DEFAULT_FIELD_NAME.as_bytes(),
+                DEFAULT_FIELD_TYPE,
+            )
             .await?;
         if updated {
             self.metrics
-                .notify_write(name, DEFAULT_FIELD_NAME.as_bytes(), DEFAULT_FIELD_TYPE)
+                .notify_write(
+                    current,
+                    name,
+                    DEFAULT_FIELD_NAME.as_bytes(),
+                    DEFAULT_FIELD_TYPE,
+                )
                 .await?;
         }
         Ok(())
@@ -630,9 +717,15 @@ impl CacheManager {
         key: &SeriesKey,
         metric_id: &MetricId,
     ) -> Result<()> {
-        let updated = self.series.update(id, key).await?;
+        let current = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap();
+        let date = SegmentDuration::date(current);
+        let updated = self.series.update(date, id, key).await?;
         if updated {
-            self.series.notify_write(id, key, metric_id).await?;
+            self.series
+                .notify_write(current, id, key, metric_id)
+                .await?;
         }
         Ok(())
     }
@@ -643,139 +736,275 @@ impl CacheManager {
         series_key: &SeriesKey,
         metric_id: &MetricId,
     ) -> Result<()> {
+        let current = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap();
+        let date = SegmentDuration::date(current);
         let updated = self
             .tag_index
-            .update(series_id, &series_key.names, &series_key.values, metric_id)
+            .update(
+                date,
+                series_id,
+                &series_key.names,
+                &series_key.values,
+                metric_id,
+            )
             .await?;
         if updated {
             self.tag_index
-                .notify_write(series_id, &series_key.names, &series_key.values, metric_id)
+                .notify_write(
+                    current,
+                    series_id,
+                    &series_key.names,
+                    &series_key.values,
+                    metric_id,
+                )
                 .await?;
         }
         Ok(())
     }
+}
+
+async fn execute_write(mut writer: CacheWriter) {
+    let mut task_queue: Vec<Task> = Vec::new();
+    let mut batch_tasks: Vec<Task>;
+    // TODO: make it configurable
+    let max_wait_time_ms: u64 = 1000;
+    let max_queue_length: usize = 16;
+
+    let task_duration = |task: &Task| -> Duration {
+        match task {
+            Task::Metric(duration, _, _, _) => *duration,
+            Task::Series(duration, _, _, _) => *duration,
+            Task::TagIndex(duration, _, _, _, _) => *duration,
+        }
+    };
 
-    async fn execute_write(mut writer: CacheWriter) {
-        // TODO: use try_recv to accumulated a number of tasks and handle them in one
-        // batch
-        while let Some(task) = writer.receiver.recv().await {
-            match task {
-                Task::Metric(name, field_name, field_type) => {
-                    let arrays: Vec<ArrayRef> = vec![
-                        Arc::new(BinaryArray::from(vec![name.to_byte_slice()])),
-                        Arc::new(UInt64Array::from(vec![hash(&name)])),
-                        Arc::new(BinaryArray::from(vec![field_name.to_byte_slice()])),
-                        Arc::new(UInt64Array::from(vec![hash(field_name.to_byte_slice())])),
-                        Arc::new(UInt8Array::from(vec![field_type])),
-                    ];
-                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
-                    writer
-                        .storage
-                        .write(WriteRequest {
-                            batch,
-                            time_range: (0..10).into(),
-                            enable_check: true,
-                        })
-                        .await
-                        .unwrap_or_else(|e| {
-                            error!("write metrics failed: {:?}", e);
-                        });
+    loop {
+        loop {
+            let wait_time: Duration = {
+                if task_queue.is_empty() {
+                    Duration::from_millis(max_wait_time_ms)
+                } else {
+                    let current = SystemTime::now()
+                        .duration_since(SystemTime::UNIX_EPOCH)
+                        .unwrap();
+                    let wait_in_ms = std::cmp::min(
+                        max_wait_time_ms
+                            - (current - task_duration(&task_queue[0])).as_millis() as u64,
+                        1,
+                    );
+                    Duration::from_millis(wait_in_ms)
                 }
-                Task::Series(id, key, metric_id) => {
-                    let offsets: Vec<i32> = vec![0, key.names.len() as i32];
-                    let tag_names_array = {
-                        let name_binary_values = key
-                            .names
-                            .iter()
-                            .map(|item| item.as_slice())
-                            .collect::<Vec<_>>();
-                        let name_binary_array = BinaryArray::from_vec(name_binary_values);
-                        ListArray::try_new(
-                            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
-                            OffsetBuffer::new(offsets.clone().into()),
-                            Arc::new(name_binary_array),
-                            None,
-                        )
-                        .unwrap()
-                    };
-
-                    let tag_values_array = {
-                        let value_binary_values = key
-                            .values
-                            .iter()
-                            .map(|item| item.as_slice())
-                            .collect::<Vec<_>>();
-                        let value_binary_array = BinaryArray::from_vec(value_binary_values);
-                        ListArray::try_new(
-                            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
-                            OffsetBuffer::new(offsets.into()),
-                            Arc::new(value_binary_array),
-                            None,
+            };
+            match timeout(wait_time, writer.receiver.recv()).await {
+                Ok(Some(task)) => {
+                    if task_queue.is_empty()
+                        || SegmentDuration::same_segment(
+                            task_duration(&task_queue[0]),
+                            task_duration(&task),
                         )
-                        .unwrap()
-                    };
-
-                    let arrays: Vec<ArrayRef> = vec![
-                        Arc::new(UInt64Array::from(vec![metric_id.0])),
-                        Arc::new(UInt64Array::from(vec![id.0])),
-                        Arc::new(tag_names_array),
-                        Arc::new(tag_values_array),
-                    ];
-                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
-                    writer
-                        .storage
-                        .write(WriteRequest {
-                            batch,
-                            time_range: (0..10).into(),
-                            enable_check: true,
-                        })
-                        .await
-                        .unwrap_or_else(|e| {
-                            error!("write metrics failed: {:?}", e);
-                        });
+                    {
+                        task_queue.push(task);
+                        if task_queue.len() >= max_queue_length {
+                            batch_tasks = std::mem::take(&mut task_queue);
+                            break;
+                        }
+                    } else {
+                        batch_tasks = std::mem::take(&mut task_queue);
+                        task_queue.push(task);
+                        break;
+                    }
                 }
-                Task::TagIndex(series_id, names, values, metric_id) => {
-                    let mut metrics_id_builder = UInt64Builder::new();
-                    let mut series_id_builder = UInt64Builder::new();
-                    let mut tag_name_builder = BinaryBuilder::new();
-                    let mut tag_value_builder = BinaryBuilder::new();
-
-                    let (names, values) = remove_default_tag(names, values);
-
-                    names.iter().zip(values.iter()).for_each(|(name, value)| {
-                        metrics_id_builder.append_value(metric_id.0);
-                        tag_name_builder.append_value(name.to_byte_slice());
-                        tag_value_builder.append_value(value.to_byte_slice());
-                        series_id_builder.append_value(series_id.0);
-                    });
-                    let arrays: Vec<ArrayRef> = vec![
-                        Arc::new(metrics_id_builder.finish()),
-                        Arc::new(tag_name_builder.finish()),
-                        Arc::new(tag_value_builder.finish()),
-                        Arc::new(series_id_builder.finish()),
-                    ];
-                    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
-                    writer
-                        .storage
-                        .write(WriteRequest {
-                            batch,
-                            time_range: (0..10).into(),
-                            enable_check: true,
-                        })
-                        .await
-                        .unwrap_or_else(|e| {
-                            error!("write metrics failed: {:?}", e);
-                        });
+                Ok(None) => {
+                    warn!("Channel closed");
+                    return;
+                }
+                Err(_) => {
+                    batch_tasks = std::mem::take(&mut task_queue);
+                    break;
                 }
             }
         }
+
+        if batch_tasks.is_empty() {
+            continue;
+        }
+
+        match &batch_tasks[0] {
+            Task::Metric(_, _, _, _) => {
+                batch_write_metrics(batch_tasks, &writer).await;
+            }
+            Task::Series(_, _, _, _) => {
+                batch_write_series(batch_tasks, &writer).await;
+            }
+            Task::TagIndex(_, _, _, _, _) => {
+                batch_write_tag_index(batch_tasks, &writer).await;
+            }
+        }
     }
 }
 
-fn remove_default_tag(
-    mut names: Vec<Vec<u8>>,
-    mut values: Vec<Vec<u8>>,
-) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
+async fn batch_write_tag_index(batch_tasks: Vec<Task>, writer: &CacheWriter) {
+    let mut metrics_id_builder = UInt64Builder::new();
+    let mut series_id_builder = UInt64Builder::new();
+    let mut field_duration_builder = UInt64Builder::new();
+    let mut tag_name_builder = BinaryBuilder::new();
+    let mut tag_value_builder = BinaryBuilder::new();
+
+    batch_tasks.into_iter().for_each(|mut task| {
+        if let Task::TagIndex(duration, series_id, ref mut names, ref mut values, metric_id) = task
+        {
+            remove_default_tag(names, values);
+
+            names.iter().zip(values.iter()).for_each(|(name, value)| {
+                metrics_id_builder.append_value(metric_id.0);
+                tag_name_builder.append_value(name.to_byte_slice());
+                tag_value_builder.append_value(value.to_byte_slice());
+                series_id_builder.append_value(series_id.0);
+                field_duration_builder.append_value(duration.as_millis() as u64);
+            });
+        } else {
+            error!("Some task are not tag index.");
+        }
+    });
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(metrics_id_builder.finish()),
+        Arc::new(tag_name_builder.finish()),
+        Arc::new(tag_value_builder.finish()),
+        Arc::new(series_id_builder.finish()),
+        Arc::new(field_duration_builder.finish()),
+    ];
+    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+    writer
+        .storage
+        .write(WriteRequest {
+            batch,
+            time_range: (0..10).into(),
+            enable_check: true,
+        })
+        .await
+        .unwrap_or_else(|e| {
+            error!("write metrics failed: {:?}", e);
+        });
+}
+
+async fn batch_write_series(batch_tasks: Vec<Task>, writer: &CacheWriter) {
+    let mut metric_id_builder = UInt64Builder::new();
+    let mut series_id_builder = UInt64Builder::new();
+    let mut field_duration_builder = UInt64Builder::new();
+
+    let mut name_binary_values: Vec<&[u8]> = Vec::new();
+    let mut value_binary_values: Vec<&[u8]> = Vec::new();
+
+    let mut offsets: Vec<i32> = vec![0; batch_tasks.len() + 1];
+    batch_tasks.iter().enumerate().for_each(|(index, task)| {
+        if let Task::Series(duration, id, key, metric_id) = task {
+            metric_id_builder.append_value(metric_id.0);
+            series_id_builder.append_value(id.0);
+            field_duration_builder.append_value(duration.as_millis() as u64);
+            key.names
+                .iter()
+                .for_each(|item| name_binary_values.push(item));
+            key.values
+                .iter()
+                .for_each(|item| value_binary_values.push(item));
+            offsets[index + 1] = offsets[index] + key.names.len() as i32;
+        } else {
+            error!("Some task are not series.");
+        }
+    });
+
+    let tag_names_array = {
+        let name_binary_array = BinaryArray::from_vec(name_binary_values);
+        ListArray::try_new(
+            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
+            OffsetBuffer::new(offsets.clone().into()),
+            Arc::new(name_binary_array),
+            None,
+        )
+        .unwrap()
+    };
+
+    let tag_values_array = {
+        let value_binary_array = BinaryArray::from_vec(value_binary_values);
+        ListArray::try_new(
+            Arc::new(Field::new(COLUMN_TAG_ITEM, DataType::Binary, true)),
+            OffsetBuffer::new(offsets.into()),
+            Arc::new(value_binary_array),
+            None,
+        )
+        .unwrap()
+    };
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(metric_id_builder.finish()),
+        Arc::new(series_id_builder.finish()),
+        Arc::new(tag_names_array),
+        Arc::new(tag_values_array),
+        Arc::new(field_duration_builder.finish()),
+    ];
+    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+    writer
+        .storage
+        .write(WriteRequest {
+            batch,
+            time_range: (0..10).into(),
+            enable_check: true,
+        })
+        .await
+        .unwrap_or_else(|e| {
+            error!("write metrics failed: {:?}", e);
+        });
+}
+
+async fn batch_write_metrics(batch_tasks: Vec<Task>, writer: &CacheWriter) {
+    let arrays: Vec<ArrayRef> = {
+        let mut metric_name_builder = BinaryBuilder::new();
+        let mut metric_id_builder = UInt64Builder::new();
+        let mut field_name_builder = BinaryBuilder::new();
+        let mut field_id_builder = UInt64Builder::new();
+        let mut field_type_builder = UInt8Builder::new();
+        let mut field_duration_builder = UInt64Builder::new();
+
+        batch_tasks.into_iter().for_each(|task| {
+            if let Task::Metric(current, name, field_name, field_type) = task {
+                metric_id_builder.append_value(hash(&name));
+                metric_name_builder.append_value(name);
+                field_id_builder.append_value(hash(field_name.to_byte_slice()));
+                field_name_builder.append_value(field_name);
+                field_type_builder.append_value(field_type);
+                field_duration_builder.append_value(current.as_millis() as u64);
+            } else {
+                error!("Some task are not metric.");
+            }
+        });
+
+        vec![
+            Arc::new(metric_name_builder.finish()),
+            Arc::new(metric_id_builder.finish()),
+            Arc::new(field_name_builder.finish()),
+            Arc::new(field_id_builder.finish()),
+            Arc::new(field_type_builder.finish()),
+            Arc::new(field_duration_builder.finish()),
+        ]
+    };
+    let batch = RecordBatch::try_new(writer.schema.clone(), arrays).unwrap();
+    writer
+        .storage
+        .write(WriteRequest {
+            batch,
+            time_range: (0..10).into(),
+            enable_check: true,
+        })
+        .await
+        .unwrap_or_else(|e| {
+            error!("write metrics failed: {:?}", e);
+        });
+}
+
+fn remove_default_tag(names: &mut Vec<Vec<u8>>, values: &mut Vec<Vec<u8>>) {
     let mut to_remove_index = HashSet::new();
     let mut index = 0;
     names.retain(|item| {
@@ -792,7 +1021,6 @@ fn remove_default_tag(
         index += 1;
         keep
     });
-    (names, values)
 }
 
 #[cfg(test)]
@@ -895,7 +1123,7 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (metric_name, field_name, field_type, filed_id, metric_id) =
+                let (metric_name, field_name, field_type, filed_id, metric_id, _) =
                     MetricsCache::parse_record_batch(&batch, 0).unwrap();
                 assert_eq!(metric_name, b"metric_neo");
                 assert_eq!(field_name, b"value");
@@ -907,7 +1135,7 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (metric_name, field_name, field_type, filed_id, metric_id) =
+                let (metric_name, field_name, field_type, filed_id, metric_id, _) =
                     MetricsCache::parse_record_batch(&batch, 0).unwrap();
                 assert_eq!(metric_name, b"metric_neo2");
                 assert_eq!(field_name, b"value");
@@ -931,7 +1159,7 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (series_id, tag_names, tag_values) =
+                let (series_id, tag_names, tag_values, _) =
                     SeriesCache::parse_record_batch(&batch, 0).await.unwrap();
 
                 assert_eq!(series_id, 11);
@@ -951,7 +1179,7 @@ mod tests {
             {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
-                let (series_id, tag_names, tag_values) =
+                let (series_id, tag_names, tag_values, _) =
                     SeriesCache::parse_record_batch(&batch, 0).await.unwrap();
 
                 assert_eq!(series_id, 22);
@@ -985,7 +1213,7 @@ mod tests {
                 let batch = item.unwrap().unwrap();
                 assert_eq!(batch.num_rows(), 3);
                 {
-                    let (metric_id, tag_name, tag_value, series_id) =
+                    let (metric_id, tag_name, tag_value, series_id, _) =
                         TagIndexCache::parse_record_batch(&batch, 0).await.unwrap();
                     assert_eq!(metric_id, 12417319948205937109);
                     assert_eq!(tag_name, b"label_a");
@@ -993,7 +1221,7 @@ mod tests {
                     assert_eq!(series_id, 11);
                 }
                 {
-                    let (metric_id, tag_name, tag_value, series_id) =
+                    let (metric_id, tag_name, tag_value, series_id, _) =
                         TagIndexCache::parse_record_batch(&batch, 1).await.unwrap();
                     assert_eq!(metric_id, 12417319948205937109);
                     assert_eq!(tag_name, b"label_b");
@@ -1001,7 +1229,7 @@ mod tests {
                     assert_eq!(series_id, 11);
                 }
                 {
-                    let (metric_id, tag_name, tag_value, series_id) =
+                    let (metric_id, tag_name, tag_value, series_id, _) =
                         TagIndexCache::parse_record_batch(&batch, 2).await.unwrap();
                     assert_eq!(metric_id, 17578343207158939466);
                     assert_eq!(tag_name, b"label_a");
@@ -1013,7 +1241,7 @@ mod tests {
                 let item = result_stream.next().await;
                 let batch = item.unwrap().unwrap();
                 {
-                    let (metric_id, tag_name, tag_value, series_id) =
+                    let (metric_id, tag_name, tag_value, series_id, _) =
                         TagIndexCache::parse_record_batch(&batch, 0).await.unwrap();
                     assert_eq!(metric_id, 17578343207158939466);
                     assert_eq!(tag_name, b"label_c");
diff --git a/src/metric_engine/src/metric/mod.rs b/src/metric_engine/src/metric/mod.rs
index bc496e8400..28342da717 100644
--- a/src/metric_engine/src/metric/mod.rs
+++ b/src/metric_engine/src/metric/mod.rs
@@ -26,7 +26,7 @@ pub struct MetricManager {
 }
 
 impl MetricManager {
-    pub async fn new(storage: TimeMergeStorageRef) -> Result<Self> {
+    pub async fn try_new(storage: TimeMergeStorageRef) -> Result<Self> {
         Ok(Self {
             inner: Arc::new(Inner {
                 storage: storage.clone(),
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index c5a6910eae..a26dfb8476 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -14,11 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-use std::{
-    collections::BTreeSet,
-    io::Write,
-    time::{Duration, SystemTime},
-};
+use std::{collections::BTreeSet, io::Write, time::Duration};
 
 pub type Result<T> = common::Result<T>;
 
@@ -42,17 +38,19 @@ pub const DEFAULT_FIELD_TYPE: u8 = 0;
 pub struct SegmentDuration(Duration);
 
 impl SegmentDuration {
-    const ONE_DAY: Duration = Duration::from_secs(24 * 60 * 60);
+    const ONE_DAY_SECOND: u64 = 24 * 60 * 60;
 
-    pub fn current_date() -> Self {
-        let now = SystemTime::now()
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs();
+    pub fn date(time: Duration) -> Self {
+        let now = time.as_secs();
         Self(Duration::from_secs(
-            now / SegmentDuration::ONE_DAY.as_secs() * SegmentDuration::ONE_DAY.as_secs(),
+            now / SegmentDuration::ONE_DAY_SECOND * SegmentDuration::ONE_DAY_SECOND,
         ))
     }
+
+    pub fn same_segment(lhs: Duration, rhs: Duration) -> bool {
+        lhs.as_secs() / SegmentDuration::ONE_DAY_SECOND
+            == rhs.as_secs() / SegmentDuration::ONE_DAY_SECOND
+    }
 }
 #[derive(PartialEq, PartialOrd, Eq, Ord, Clone)]
 pub struct Label {

From 5bf5060a68ccb0cc0da51878ab259ea495615da2 Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Sat, 25 Jan 2025 01:57:58 -0600
Subject: [PATCH 7/9] add index cache layer

---
 src/metric_engine/src/index/cache.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index 3d011b3817..dad38d9d42 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -539,13 +539,13 @@ impl TagIndexCache {
                 let mut tag_values = tag_values.clone();
                 remove_default_tag(&mut tag_names, &mut tag_values);
                 tag_names
-                    .iter()
-                    .zip(tag_values.iter())
+                    .into_iter()
+                    .zip(tag_values.into_iter())
                     .for_each(|(name, value)| {
                         cache_guard
-                            .entry(name.clone())
+                            .entry(name)
                             .or_default()
-                            .entry(value.clone())
+                            .entry(value)
                             .or_default()
                             .entry(*metric_id)
                             .or_default()

From b54bcad5daeaaa5c3b668b3bcbb58ff3c6929d0f Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Sat, 25 Jan 2025 02:25:18 -0600
Subject: [PATCH 8/9] fix ci

---
 src/metric_engine/src/index/cache.rs | 2 ++
 src/metric_engine/src/types.rs       | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index dad38d9d42..0ce3c2c2e8 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -14,6 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 use std::{
     collections::{HashMap, HashSet},
     path::Path,
@@ -105,6 +106,7 @@ impl MetricsCache {
         }
     }
 
+    #[allow(clippy::type_complexity)]
     fn parse_record_batch(
         batch: &RecordBatch,
         index: usize,
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index a26dfb8476..3d0b446fc5 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -14,6 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 use std::{collections::BTreeSet, io::Write, time::Duration};
 
 pub type Result<T> = common::Result<T>;

From 3f47a80891b2b191438ed1bbedf112f27f9b0bbb Mon Sep 17 00:00:00 2001
From: neochen <neochen428@gmail.com>
Date: Sat, 25 Jan 2025 11:21:49 -0600
Subject: [PATCH 9/9] bug fix

---
 src/metric_engine/src/index/cache.rs | 97 +++++++++++++++++++---------
 src/metric_engine/src/types.rs       | 12 +++-
 2 files changed, 78 insertions(+), 31 deletions(-)

diff --git a/src/metric_engine/src/index/cache.rs b/src/metric_engine/src/index/cache.rs
index 0ce3c2c2e8..08624ef497 100644
--- a/src/metric_engine/src/index/cache.rs
+++ b/src/metric_engine/src/index/cache.rs
@@ -854,22 +854,36 @@ async fn batch_write_tag_index(batch_tasks: Vec<Task>, writer: &CacheWriter) {
     let mut tag_name_builder = BinaryBuilder::new();
     let mut tag_value_builder = BinaryBuilder::new();
 
-    batch_tasks.into_iter().for_each(|mut task| {
-        if let Task::TagIndex(duration, series_id, ref mut names, ref mut values, metric_id) = task
-        {
-            remove_default_tag(names, values);
-
-            names.iter().zip(values.iter()).for_each(|(name, value)| {
-                metrics_id_builder.append_value(metric_id.0);
-                tag_name_builder.append_value(name.to_byte_slice());
-                tag_value_builder.append_value(value.to_byte_slice());
-                series_id_builder.append_value(series_id.0);
-                field_duration_builder.append_value(duration.as_millis() as u64);
-            });
-        } else {
-            error!("Some task are not tag index.");
-        }
-    });
+    let mut start_ts: i64 = 0;
+    let mut end_ts: i64 = 0;
+    let task_len = batch_tasks.len();
+
+    batch_tasks
+        .into_iter()
+        .enumerate()
+        .for_each(|(index, mut task)| {
+            if let Task::TagIndex(duration, series_id, ref mut names, ref mut values, metric_id) =
+                task
+            {
+                if index == 0 {
+                    start_ts = duration.as_millis() as i64;
+                } else if index == task_len - 1 {
+                    end_ts = duration.as_millis() as i64;
+                }
+
+                remove_default_tag(names, values);
+
+                names.iter().zip(values.iter()).for_each(|(name, value)| {
+                    metrics_id_builder.append_value(metric_id.0);
+                    tag_name_builder.append_value(name.to_byte_slice());
+                    tag_value_builder.append_value(value.to_byte_slice());
+                    series_id_builder.append_value(series_id.0);
+                    field_duration_builder.append_value(duration.as_millis() as u64);
+                });
+            } else {
+                error!("Some task are not tag index.");
+            }
+        });
 
     let arrays: Vec<ArrayRef> = vec![
         Arc::new(metrics_id_builder.finish()),
@@ -883,7 +897,7 @@ async fn batch_write_tag_index(batch_tasks: Vec<Task>, writer: &CacheWriter) {
         .storage
         .write(WriteRequest {
             batch,
-            time_range: (0..10).into(),
+            time_range: (Timestamp(start_ts)..Timestamp(end_ts)).into(),
             enable_check: true,
         })
         .await
@@ -900,9 +914,19 @@ async fn batch_write_series(batch_tasks: Vec<Task>, writer: &CacheWriter) {
     let mut name_binary_values: Vec<&[u8]> = Vec::new();
     let mut value_binary_values: Vec<&[u8]> = Vec::new();
 
+    let mut start_ts: i64 = 0;
+    let mut end_ts: i64 = 0;
+    let task_len = batch_tasks.len();
+
     let mut offsets: Vec<i32> = vec![0; batch_tasks.len() + 1];
     batch_tasks.iter().enumerate().for_each(|(index, task)| {
         if let Task::Series(duration, id, key, metric_id) = task {
+            if index == 0 {
+                start_ts = duration.as_millis() as i64;
+            } else if index == task_len - 1 {
+                end_ts = duration.as_millis() as i64;
+            }
+
             metric_id_builder.append_value(metric_id.0);
             series_id_builder.append_value(id.0);
             field_duration_builder.append_value(duration.as_millis() as u64);
@@ -952,7 +976,7 @@ async fn batch_write_series(batch_tasks: Vec<Task>, writer: &CacheWriter) {
         .storage
         .write(WriteRequest {
             batch,
-            time_range: (0..10).into(),
+            time_range: (Timestamp(start_ts)..Timestamp(end_ts)).into(),
             enable_check: true,
         })
         .await
@@ -970,18 +994,31 @@ async fn batch_write_metrics(batch_tasks: Vec<Task>, writer: &CacheWriter) {
         let mut field_type_builder = UInt8Builder::new();
         let mut field_duration_builder = UInt64Builder::new();
 
-        batch_tasks.into_iter().for_each(|task| {
-            if let Task::Metric(current, name, field_name, field_type) = task {
-                metric_id_builder.append_value(hash(&name));
-                metric_name_builder.append_value(name);
-                field_id_builder.append_value(hash(field_name.to_byte_slice()));
-                field_name_builder.append_value(field_name);
-                field_type_builder.append_value(field_type);
-                field_duration_builder.append_value(current.as_millis() as u64);
-            } else {
-                error!("Some task are not metric.");
-            }
-        });
+        let mut start_ts: i64 = 0;
+        let mut end_ts: i64 = 0;
+        let task_len = batch_tasks.len();
+
+        batch_tasks
+            .into_iter()
+            .enumerate()
+            .for_each(|(index, task)| {
+                if let Task::Metric(duration, name, field_name, field_type) = task {
+                    if index == 0 {
+                        start_ts = duration.as_millis() as i64;
+                    } else if index == task_len - 1 {
+                        end_ts = duration.as_millis() as i64;
+                    }
+
+                    metric_id_builder.append_value(hash(&name));
+                    metric_name_builder.append_value(name);
+                    field_id_builder.append_value(hash(field_name.to_byte_slice()));
+                    field_name_builder.append_value(field_name);
+                    field_type_builder.append_value(field_type);
+                    field_duration_builder.append_value(duration.as_millis() as u64);
+                } else {
+                    error!("Some task are not metric.");
+                }
+            });
 
         vec![
             Arc::new(metric_name_builder.finish()),
diff --git a/src/metric_engine/src/types.rs b/src/metric_engine/src/types.rs
index 3d0b446fc5..89effc0d97 100644
--- a/src/metric_engine/src/types.rs
+++ b/src/metric_engine/src/types.rs
@@ -15,7 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::{collections::BTreeSet, io::Write, time::Duration};
+use std::{
+    collections::{BTreeSet, HashMap, HashSet},
+    io::Write,
+    time::Duration,
+};
 
 pub type Result<T> = common::Result<T>;
 
@@ -131,3 +135,9 @@ impl SeriesKey {
 pub fn hash(buf: &[u8]) -> u64 {
     seahash::hash(buf)
 }
+
+pub type MetricFieldMap = HashMap<MetricName, (FieldName, FieldType)>;
+pub type SeriesKeyMap = HashMap<SeriesId, SeriesKey>;
+pub type MetricSeriesMap = HashMap<MetricId, HashSet<SeriesId>>;
+pub type TagValueMap = HashMap<TagValue, MetricSeriesMap>;
+pub type TagIndexMap = HashMap<TagName, TagValueMap>;