[PLAT-440] roll up routing in metrics view

docs/docs/reference/project-files/metrics-views.md

@@ -264,6 +264,48 @@ _[array of object]_ - Used to define annotations that can be displayed on charts
 
       - **`exclude`** - _[object]_ - Select all fields except those listed here 
 
+### `rollups`
+
+_[array of object]_ - Pre-aggregated rollup tables that can be used to accelerate queries. When a query's dimensions, measures, time grain, and time range match a rollup, the query is automatically routed to the rollup table instead of the base table. 
+
+  - **`model`** - _[string]_ - Refers to the model or table powering the rollup (required) _(required)_
+
+  - **`database`** - _[string]_ - Refers to the database to use in the OLAP engine 
+
+  - **`database_schema`** - _[string]_ - Refers to the schema to use in the OLAP engine 
+
+  - **`time_grain`** - _[string]_ - The time grain of the rollup (required). Valid values are: millisecond, second, minute, hour, day, week, month, quarter, year _(required)_
+
+  - **`time_zone`** - _[string]_ - IANA timezone of the rollup table (e.g. America/New_York). For day+ grains, queries are only routed to the rollup if the query timezone matches. 
+
+  - **`dimensions`** - _[oneOf]_ - Optional field selectors for dimensions to include in the rollup from the base metrics view. If not specified, all dimensions are included. 
+
+    - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection
+
+    - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection
+
+    - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion
+
+      - **`regex`** - _[string]_ - Select fields using a regular expression 
+
+      - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic 
+
+      - **`exclude`** - _[object]_ - Select all fields except those listed here 
+
+  - **`measures`** - _[oneOf]_ - Optional field selectors for measures to include in the rollup from the base metrics view. If not specified, all measures are included. 
+
+    - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection
+
+    - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection
+
+    - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion
+
+      - **`regex`** - _[string]_ - Select fields using a regular expression 
+
+      - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic 
+
+      - **`exclude`** - _[object]_ - Select all fields except those listed here 
+
 ### `security`
 
 _[object]_ - Defines [security rules and access control policies](/developers/build/metrics-view/security) for resources 

proto/gen/rill/runtime/v1/runtime.swagger.yaml

@@ -3806,6 +3806,42 @@ definitions:
         type: string
         title: Raw SQL expression to use as the frame (starting with "ROWS ...")
     title: Window to apply to the measure calculation
+  MetricsViewSpecRollup:
+    type: object
+    properties:
+      database:
+        type: string
+      databaseSchema:
+        type: string
+      table:
+        type: string
+      model:
+        type: string
+      timeGrain:
+        $ref: '#/definitions/v1TimeGrain'
+        description: Time grain of the rollup.
+      timeZone:
+        type: string
+        title: IANA timezone the rollup was aggregated in; defaults to UTC
+      dimensions:
+        type: array
+        items:
+          type: string
+        description: Resolved list of dimensions present in the rollup. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
+      measures:
+        type: array
+        items:
+          type: string
+        description: Resolved list of measures present in the rollup. If `measures_selector` is set, this will only be set in `state.valid_spec`.
+      dimensionsSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+      measuresSelector:
+        $ref: '#/definitions/v1FieldSelector'
+        description: Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+    description: |-
+      Pre-aggregated rollup that can be used to accelerate queries.
+      The system automatically routes queries to a rollup when the query can be satisfied from the pre-aggregated data.
   NumericHistogramBinsBin:
     type: object
     properties:
@@ -6411,19 +6447,33 @@ definitions:
         description: Month number to use as the base for time aggregations by year. Defaults to 1 (January).
       cacheEnabled:
         type: boolean
-        description: Cache controls for the metrics view.
+        description: Cache controls for the metrics view. By default, enabled for Rill managed models and disabled for streaming (externally managed) data sources.
       cacheKeySql:
         type: string
+        description: Defaults to use watermark if cache is enabled.
       cacheKeyTtlSeconds:
         type: string
         format: int64
+        description: Defaults to 60 seconds if cache is enabled.
+      cacheTimestampsTtlSeconds:
+        type: string
+        format: int64
+        description: |-
+          TTL for caching timestamp boundaries (min/max) of base and rollup tables. Defaults to 5 minutes if unset.
+          Takes effect only when mv level caching is disabled otherwise the timestamp boundaries are cached with the same TTL
+          as the rest of the query results set via `cache.timestamps_ttl` in the metrics view YAML.
       queryAttributes:
         type: object
         additionalProperties:
           type: string
         description: |-
           Query attributes that can be templated with user context and used by drivers (e.g., appended to SETTINGS in ClickHouse).
           Keys and values are stored as templates and will be resolved at query time.
+      rollups:
+        type: array
+        items:
+          type: object
+          $ref: '#/definitions/MetricsViewSpecRollup'
   v1MetricsViewSpecAnnotation:
     type: object
     properties:

proto/rill/runtime/v1/resources.proto

@@ -299,6 +299,26 @@ message MetricsViewSpec {
     // Signifies that the underlying table has `duration` column. Will be used while querying to add additional filter.
     bool has_duration = 11;
   }
+  // Pre-aggregated rollup that can be used to accelerate queries.
+  // The system automatically routes queries to a rollup when the query can be satisfied from the pre-aggregated data.
+  message Rollup {
+    string database = 1;
+    string database_schema = 2;
+    string table = 3;
+    string model = 4;
+    // Time grain of the rollup.
+    TimeGrain time_grain = 5;
+    // IANA timezone the rollup was aggregated in; defaults to UTC
+    string time_zone = 6;
+    // Resolved list of dimensions present in the rollup. If `dimensions_selector` is set, this will only be set in `state.valid_spec`.
+    repeated string dimensions = 7;
+    // Resolved list of measures present in the rollup. If `measures_selector` is set, this will only be set in `state.valid_spec`.
+    repeated string measures = 8;
+    // Dynamic selector for `dimensions`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+    FieldSelector dimensions_selector = 9;
+    // Dynamic selector for `measures`. Will be processed during validation, so it will always be empty in `state.valid_spec`.
+    FieldSelector measures_selector = 10;
+  }
   // Connector containing the table
   string connector = 1;
   // Name of the database where table is located (optional)
@@ -338,13 +358,20 @@ message MetricsViewSpec {
   uint32 first_day_of_week = 12;
   // Month number to use as the base for time aggregations by year. Defaults to 1 (January).
   uint32 first_month_of_year = 13;
-  // Cache controls for the metrics view.
+  // Cache controls for the metrics view. By default, enabled for Rill managed models and disabled for streaming (externally managed) data sources.
   optional bool cache_enabled = 25;
+  // Defaults to use watermark if cache is enabled.
   string cache_key_sql = 26;
+  // Defaults to 60 seconds if cache is enabled.
   int64 cache_key_ttl_seconds = 27;
+  // TTL for caching timestamp boundaries (min/max) of base and rollup tables. Defaults to 5 minutes if unset.
+  // Takes effect only when mv level caching is disabled otherwise the timestamp boundaries are cached with the same TTL
+  // as the rest of the query results set via `cache.timestamps_ttl` in the metrics view YAML.
+  int64 cache_timestamps_ttl_seconds = 35;
   // Query attributes that can be templated with user context and used by drivers (e.g., appended to SETTINGS in ClickHouse).
   // Keys and values are stored as templates and will be resolved at query time.
   map<string, string> query_attributes = 33;
+  repeated Rollup rollups = 34;
 }
 
 message SecurityRule {