diff --git a/src/common/column/src/types/native.rs b/src/common/column/src/types/native.rs index 0ce6dfc1b42cd..74a4aa8b30295 100644 --- a/src/common/column/src/types/native.rs +++ b/src/common/column/src/types/native.rs @@ -440,7 +440,6 @@ impl Neg for months_days_micros { } } -/// The in-memory representation of the MonthDayNano variant of the "Interval" logical type. #[derive( Debug, Copy, @@ -460,12 +459,12 @@ pub struct timestamp_tz(pub i128); impl Hash for timestamp_tz { fn hash(&self, state: &mut H) { - self.total_micros().hash(state) + self.timestamp().hash(state) } } impl PartialEq for timestamp_tz { fn eq(&self, other: &Self) -> bool { - self.total_micros() == other.total_micros() + self.timestamp() == other.timestamp() } } impl PartialOrd for timestamp_tz { @@ -476,17 +475,18 @@ impl PartialOrd for timestamp_tz { impl Ord for timestamp_tz { fn cmp(&self, other: &Self) -> Ordering { - let total_micros = self.total_micros(); - let other_micros = other.total_micros(); - total_micros.cmp(&other_micros) + let timestamp = self.timestamp(); + let other_micros = other.timestamp(); + timestamp.cmp(&other_micros) } } impl timestamp_tz { pub const MICROS_PER_SECOND: i64 = 1_000_000; + #[inline] pub fn new(timestamp: i64, offset: i32) -> Self { - let ts = timestamp as u64 as i128; // <- 中间加一次 u64 屏蔽符号位 + let ts = timestamp as u64 as i128; let off = (offset as i128) << 64; Self(off | ts) } @@ -507,32 +507,19 @@ impl timestamp_tz { } #[inline] - pub fn hours_offset(&self) -> i8 { - (self.seconds_offset() / 3600) as i8 + pub fn micros_offset_inner(seconds: i64) -> Option { + seconds.checked_mul(Self::MICROS_PER_SECOND) } #[inline] - pub fn total_micros(&self) -> i64 { - self.try_total_micros().unwrap_or_else(|| { - error!( - "interval is out of range: timestamp={}, offset={}", - self.timestamp(), - self.seconds_offset() - ); - 0 - }) - } - - #[inline] - pub fn try_total_micros(&self) -> Option { - let offset_micros = self.micros_offset()?; - self.timestamp().checked_sub(offset_micros) + pub fn hours_offset(&self) -> i8 { + (self.seconds_offset() / 3600) as i8 } } impl Display for timestamp_tz { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let timestamp = Timestamp::from_microsecond(self.total_micros()).unwrap(); + let timestamp = Timestamp::from_microsecond(self.timestamp()).unwrap(); let offset = tz::Offset::from_seconds(self.seconds_offset()).unwrap(); let string = strtime::format( diff --git a/src/query/expression/src/row_encoding.rs b/src/query/expression/src/row_encoding.rs index 3eacc71a99d01..71e80470802dc 100644 --- a/src/query/expression/src/row_encoding.rs +++ b/src/query/expression/src/row_encoding.rs @@ -13,6 +13,7 @@ // limitations under the License. use databend_common_column::types::months_days_micros; +use databend_common_column::types::timestamp_tz; use crate::types::i256; use crate::types::F32; @@ -103,3 +104,11 @@ impl FixedLengthEncoding for months_days_micros { self.total_micros().encode() } } + +impl FixedLengthEncoding for timestamp_tz { + type Encoded = [u8; 8]; + + fn encode(self) -> [u8; 8] { + self.timestamp().encode() + } +} diff --git a/src/query/expression/src/types/timestamp_tz.rs b/src/query/expression/src/types/timestamp_tz.rs index 33ba974d76e9e..451eac8e27d59 100644 --- a/src/query/expression/src/types/timestamp_tz.rs +++ b/src/query/expression/src/types/timestamp_tz.rs @@ -148,14 +148,39 @@ pub fn string_to_timestamp_tz<'a, F: FnOnce() -> &'a TimeZone>( .or_else(|_| fmt::strtime::parse("%Y-%m-%d %H:%M:%S%.f %z", ts_str)) .or_else(|_| fmt::strtime::parse("%Y-%m-%d %H:%M:%S%.f %:z", ts_str)) .or_else(|_| fmt::strtime::parse("%Y-%m-%d %H:%M:%S%.f", ts_str))?; - let datetime = time.to_datetime()?; - let timestamp = tz::offset(0).to_timestamp(datetime)?; - let offset = time - .offset() - .unwrap_or_else(|| fn_tz().to_offset(timestamp)); - - Ok(timestamp_tz::new( - timestamp.as_microsecond(), - offset.seconds(), - )) + match time.offset() { + None => { + let datetime = time.to_datetime()?; + let timestamp = tz::offset(0).to_timestamp(datetime)?; + let offset = fn_tz().to_offset(timestamp); + + Ok(timestamp_tz::new( + timestamp.as_microsecond() - (offset.seconds() as i64 * 1_000_000), + offset.seconds(), + )) + } + Some(offset) => { + let timestamp = time.to_timestamp()?; + + Ok(timestamp_tz::new( + timestamp.as_microsecond(), + offset.seconds(), + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stores_utc_in_timestamp_field() { + let tz = TimeZone::get("Asia/Shanghai").unwrap(); + let value = string_to_timestamp_tz(b"2021-12-20 17:01:01 +0800", || &tz).expect("parse tz"); + assert_eq!(value.seconds_offset(), 28_800); + // timestamp() keeps the UTC instant (09:01:01). + assert_eq!(value.timestamp(), 1_639_990_861_000_000); + assert_eq!(value.to_string(), "2021-12-20 17:01:01.000000 +0800"); + } } diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index bff2a202ab780..591b5c9aa02c2 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -201,8 +201,8 @@ fn timestamp_tz_domain_to_timestamp_domain( domain: &SimpleDomain, ) -> Option> { Some(SimpleDomain { - min: domain.min.total_micros(), - max: domain.max.total_micros(), + min: domain.min.timestamp(), + max: domain.max.timestamp(), }) } @@ -689,7 +689,10 @@ fn register_timestamp_to_timestamp_tz(registry: &mut FunctionRegistry) { } }; let offset = ctx.func_ctx.tz.to_offset(ts); - let ts_tz = timestamp_tz::new(val, offset.seconds()); + let ts_tz = timestamp_tz::new( + val - (offset.seconds() as i64 * 1_000_000), + offset.seconds(), + ); output.push(ts_tz) })(val, ctx) @@ -712,7 +715,7 @@ fn register_timestamp_tz_to_timestamp(registry: &mut FunctionRegistry) { ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_1_arg::(|val, output, _ctx| { - output.push(val.total_micros()) + output.push(val.timestamp()) })(val, ctx) } } diff --git a/src/query/functions/src/scalars/timestamp/src/interval.rs b/src/query/functions/src/scalars/timestamp/src/interval.rs index a22e4fc53814e..6cfa5bfda730d 100644 --- a/src/query/functions/src/scalars/timestamp/src/interval.rs +++ b/src/query/functions/src/scalars/timestamp/src/interval.rs @@ -145,7 +145,9 @@ fn register_interval_add_sub_mul(registry: &mut FunctionRegistry) { return; } }; - eval_timestamp_plus(a, b, output, ctx, |input| input.timestamp(), |result| timestamp_tz::new(result, a.seconds_offset()), TimeZone::fixed(offset)); + eval_timestamp_plus(a, b, output, ctx, |input| input.timestamp(), |result| { + timestamp_tz::new(result, a.seconds_offset()) + }, TimeZone::fixed(offset)); }, ), ); @@ -183,7 +185,9 @@ fn register_interval_add_sub_mul(registry: &mut FunctionRegistry) { return; } }; - eval_timestamp_plus(a, b, output, ctx, |input| input.timestamp(), |result| timestamp_tz::new(result, a.seconds_offset()), TimeZone::fixed(offset)); + eval_timestamp_plus(a, b, output, ctx, |input| input.timestamp(), |result| { + timestamp_tz::new(result, a.seconds_offset()) + }, TimeZone::fixed(offset)); }, ), ); @@ -235,7 +239,9 @@ fn register_interval_add_sub_mul(registry: &mut FunctionRegistry) { return; } }; - eval_timestamp_minus(a, b, output, ctx, |input| input.timestamp(), |result| timestamp_tz::new(result, a.seconds_offset()), TimeZone::fixed(offset)); + eval_timestamp_minus(a, b, output, ctx, |input| input.timestamp(), |result| { + timestamp_tz::new(result, a.seconds_offset()) + }, TimeZone::fixed(offset)); }, ), ); diff --git a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs index ca5e044531140..4e229dad9e40c 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sorts/core/row_convert/variable.rs @@ -17,12 +17,14 @@ use std::ops::Range; use databend_common_column::bitmap::Bitmap; use databend_common_column::buffer::Buffer; use databend_common_column::types::months_days_micros; +use databend_common_column::types::timestamp_tz; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::binary::BinaryColumn; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::i256; use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::types::AccessType; use databend_common_expression::types::BinaryType; use databend_common_expression::types::BooleanType; @@ -128,6 +130,7 @@ impl RowConverter for VariableRowConverter { | DataType::Number(_) | DataType::Decimal(_) | DataType::Timestamp + | DataType::TimestampTz | DataType::Interval | DataType::Date | DataType::Binary @@ -337,6 +340,11 @@ impl LengthCalculatorVisitor<'_> { *length += i64::ENCODED_LEN } } + DataType::TimestampTz => { + for length in self.lengths.iter_mut() { + *length += timestamp_tz::ENCODED_LEN + } + } DataType::Date => { for length in self.lengths.iter_mut() { *length += i32::ENCODED_LEN @@ -570,6 +578,21 @@ impl EncodeVisitor<'_> { self.field.nulls_first, ); } + DataType::TimestampTz => { + let scalar_value = if is_null { + timestamp_tz::default() + } else { + *scalar.as_timestamp_tz().unwrap() + }; + fixed_encode_const::( + &mut self.out.data, + &mut self.out.offsets, + is_null, + scalar_value, + self.field.asc, + self.field.nulls_first, + ); + } DataType::Date => { let scalar_value = if is_null { 0i32 diff --git a/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test b/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test index fd811caeeeeb0..58ed6366456c5 100644 --- a/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test +++ b/tests/sqllogictests/suites/base/11_data_type/11_0001_data_type_date_time.test @@ -336,6 +336,20 @@ insert into t values(1, '2022-02-03T03:02:00+0800') statement ok drop table t +query TT +WITH t AS ( + SELECT to_timestamp_tz('2021-12-20 09:00:01 +00:00') AS v, 'A_09:00@UTC' AS tag + UNION ALL + SELECT to_timestamp_tz('2021-12-20 17:00:00 +08:00'), 'B_17:00@+8' + UNION ALL + SELECT to_timestamp_tz('2021-12-20 09:30:00 +00:00'), 'C_09:30@UTC' +) +SELECT tag, v FROM t ORDER BY v; +---- +B_17:00@+8 2021-12-20 17:00:00.000000 +0800 +A_09:00@UTC 2021-12-20 09:00:01.000000 +0000 +C_09:30@UTC 2021-12-20 09:30:00.000000 +0000 + statement ok set timezone='UTC'